diff --git a/.gitignore b/.gitignore index 64ac199e..42117323 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ build/ !*/build/intermediates/ local.properties .gradle/ +.kotlin/ captures/ .externalNativeBuild/ debug_keystore/ @@ -40,6 +41,11 @@ dependency-reduced-pom.xml # Linters .lint/ +# Python test tooling +**/__pycache__/ +*.py[cod] +release-gate-results/ + # Other *.log .cxx/ diff --git a/app/src/test/kotlin/com/bitchat/android/contracts/ClientRewriteWireContractTest.kt b/app/src/test/kotlin/com/bitchat/android/contracts/ClientRewriteWireContractTest.kt new file mode 100644 index 00000000..d1381170 --- /dev/null +++ b/app/src/test/kotlin/com/bitchat/android/contracts/ClientRewriteWireContractTest.kt @@ -0,0 +1,301 @@ +package com.bitchat.android.contracts + +import com.bitchat.android.model.BitchatFilePacket +import com.bitchat.android.model.BitchatMessage +import com.bitchat.android.model.FragmentPayload +import com.bitchat.android.model.IdentityAnnouncement +import com.bitchat.android.model.NoisePayload +import com.bitchat.android.model.NoisePayloadType +import com.bitchat.android.model.PeerCapabilities +import com.bitchat.android.model.PrivateMessagePacket +import com.bitchat.android.model.RequestSyncPacket +import com.bitchat.android.model.UnknownAnnouncementTLV +import com.bitchat.android.protocol.BinaryProtocol +import com.bitchat.android.protocol.BitchatPacket +import com.bitchat.android.protocol.MessageType +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import java.util.Date + +/** + * Golden wire vectors for formats that a from-scratch client must reproduce. + * + * These assertions deliberately compare literal bytes rather than relying only + * on encode/decode round trips, which can hide matching bugs in both methods. + */ +class ClientRewriteWireContractTest { + + @Test + fun `v1 packet matches canonical unpadded bytes`() { + val packet = BitchatPacket( + version = 1u, + type = MessageType.MESSAGE.value, + senderID = hex("1011121314151617"), + recipientID = null, + timestamp = 0x0102030405060708uL, + payload = hex("aabbcc"), + signature = null, + ttl = 7u + ) + + val encoded = BinaryProtocol.encode(packet, padding = false) + + assertArrayEquals( + hex("01020701020304050607080000031011121314151617aabbcc"), + encoded + ) + assertEquals(packet, BinaryProtocol.decode(encoded!!)) + } + + @Test + fun `v2 routed signed packet matches canonical section order`() { + val signature = ByteArray(64) { 0x5a } + val packet = BitchatPacket( + version = 2u, + type = MessageType.NOISE_ENCRYPTED.value, + senderID = hex("0102030405060708"), + recipientID = hex("1112131415161718"), + timestamp = 42uL, + payload = hex("dead"), + signature = signature, + ttl = 5u, + route = listOf( + hex("2122232425262728"), + hex("3132333435363738") + ) + ) + + val encoded = BinaryProtocol.encode(packet, padding = false)!! + val expectedPrefix = hex( + "021105000000000000002a0b00000002" + + "0102030405060708" + + "1112131415161718" + + "02" + + "2122232425262728" + + "3132333435363738" + + "dead" + ) + + assertArrayEquals(expectedPrefix + signature, encoded) + assertEquals(packet, BinaryProtocol.decode(encoded)) + } + + @Test + fun `minimal chat message matches canonical binary payload`() { + val message = BitchatMessage( + id = "id", + sender = "bob", + content = "hi", + timestamp = Date(0x0102030405060708L) + ) + + val encoded = message.toBinaryPayload() + + assertArrayEquals( + hex("00010203040506070802696403626f6200026869"), + encoded + ) + assertEquals(message, BitchatMessage.fromBinaryPayload(encoded!!)) + } + + @Test + fun `chat message optional fields use flags and UTF-8 byte lengths`() { + val message = BitchatMessage( + id = "m", + sender = "é", + content = "hello", + timestamp = Date(42L), + isRelay = true, + originalSender = "o", + isPrivate = true, + recipientNickname = "r", + senderPeerID = "p", + mentions = listOf("a", "β"), + channel = "c" + ) + + val encoded = message.toBinaryPayload()!! + + assertArrayEquals( + hex( + "7f000000000000002a" + + "016d" + + "02c3a9" + + "000568656c6c6f" + + "016f" + + "0172" + + "0170" + + "02" + + "0161" + + "02ceb2" + + "0163" + ), + encoded + ) + assertEquals(message, BitchatMessage.fromBinaryPayload(encoded)) + } + + @Test + fun `encrypted chat payload carries ciphertext instead of placeholder content`() { + val message = BitchatMessage( + id = "e", + sender = "alice", + content = "must-not-be-on-wire", + timestamp = Date(1L), + encryptedContent = hex("000102ff"), + isEncrypted = true, + isPrivate = true + ) + + val decoded = BitchatMessage.fromBinaryPayload(message.toBinaryPayload()!!)!! + + assertEquals("", decoded.content) + assertArrayEquals(hex("000102ff"), decoded.encryptedContent) + assertTrue(decoded.isEncrypted) + assertTrue(decoded.isPrivate) + assertFalse(message.toBinaryPayload()!!.toString(Charsets.ISO_8859_1).contains("must-not-be-on-wire")) + } + + @Test + fun `private message and Noise envelopes match deployed type bytes`() { + val privateMessage = PrivateMessagePacket(messageID = "m1", content = "hi") + val privateMessageBytes = hex("00026d3101026869") + + assertArrayEquals(privateMessageBytes, privateMessage.encode()) + assertEquals(privateMessage, PrivateMessagePacket.decode(privateMessageBytes)) + assertArrayEquals( + hex("0100026d3101026869"), + NoisePayload(NoisePayloadType.PRIVATE_MESSAGE, privateMessageBytes).encode() + ) + assertEquals( + NoisePayloadType.FILE_TRANSFER, + NoisePayload.decode(hex("09cafe"))?.type + ) + assertArrayEquals( + hex("20cafe"), + NoisePayload.decode(hex("09cafe"))!!.encode() + ) + } + + @Test + fun `fragment payload matches the thirteen byte iOS header`() { + val fragment = FragmentPayload( + fragmentID = hex("0001020304050607"), + index = 1, + total = 3, + originalType = MessageType.MESSAGE.value, + data = hex("aabb") + ) + val wire = hex("00010203040506070001000302aabb") + + assertArrayEquals(wire, fragment.encode()) + assertEquals(fragment, FragmentPayload.decode(wire)) + assertTrue(fragment.isValid()) + } + + @Test + fun `sync request matches canonical TLV bytes and skips extensions`() { + val request = RequestSyncPacket( + p = 19, + m = 0x01020304L, + data = hex("aabb") + ) + val wire = hex("0100011302000401020304030002aabb") + + assertArrayEquals(wire, request.encode()) + assertSyncRequestEquals(request, RequestSyncPacket.decode(wire)) + + val withExtension = hex("7f0002cafe") + wire + assertSyncRequestEquals(request, RequestSyncPacket.decode(withExtension)) + } + + @Test + fun `identity announcement matches canonical TLV order and preserves extensions`() { + val announcement = IdentityAnnouncement( + nickname = "bob", + noisePublicKey = ByteArray(32) { 0x11 }, + signingPublicKey = ByteArray(32) { 0x22 }, + capabilities = PeerCapabilities.PRIVATE_MEDIA, + unknownTLVs = listOf(UnknownAnnouncementTLV(0x7f, hex("cafe"))) + ) + val expected = + hex("0103626f620220") + + ByteArray(32) { 0x11 } + + hex("0320") + + ByteArray(32) { 0x22 } + + hex("050200017f02cafe") + + val encoded = announcement.encode() + + assertArrayEquals(expected, encoded) + assertEquals(announcement, IdentityAnnouncement.decode(encoded!!)) + } + + @Test + fun `file transfer matches deployed mixed-width TLV vector`() { + val packet = BitchatFilePacket( + fileName = "a", + fileSize = 2, + mimeType = "m", + content = hex("dead") + ) + val wire = hex("01000161020004000000020300016d0400000002dead") + + assertArrayEquals(wire, packet.encode()) + + val decoded = BitchatFilePacket.decode(wire) + assertNotNull(decoded) + assertEquals(packet.fileName, decoded!!.fileName) + assertEquals(packet.fileSize, decoded.fileSize) + assertEquals(packet.mimeType, decoded.mimeType) + assertArrayEquals(packet.content, decoded.content) + } + + @Test + fun `required message prefixes reject every truncation`() { + val wire = BitchatMessage( + id = "id", + sender = "bob", + content = "hello", + timestamp = Date(1L) + ).toBinaryPayload()!! + + for (length in 0 until wire.size) { + assertNull( + "Accepted required message prefix of $length/${wire.size} bytes", + BitchatMessage.fromBinaryPayload(wire.copyOf(length)) + ) + } + assertNotNull(BitchatMessage.fromBinaryPayload(wire)) + } + + @Test + fun `TLV decoders reject missing required fields and truncated values`() { + assertNull(PrivateMessagePacket.decode(hex("00026d31"))) + assertNull(PrivateMessagePacket.decode(hex("00026d3101036869"))) + assertNull(RequestSyncPacket.decode(hex("0100011302000401020304"))) + assertNull(IdentityAnnouncement.decode(hex("0103626f62022011"))) + assertNull(BitchatFilePacket.decode(hex("010001610400000002de"))) + assertNull(FragmentPayload.decode(ByteArray(FragmentPayload.HEADER_SIZE - 1))) + } + + private fun hex(value: String): ByteArray { + require(value.length % 2 == 0) + return value.chunked(2).map { it.toInt(16).toByte() }.toByteArray() + } + + private fun assertSyncRequestEquals( + expected: RequestSyncPacket, + actual: RequestSyncPacket? + ) { + assertNotNull(actual) + assertEquals(expected.p, actual!!.p) + assertEquals(expected.m, actual.m) + assertArrayEquals(expected.data, actual.data) + } +} diff --git a/app/src/test/kotlin/com/bitchat/android/noise/NoiseExternalVectorTest.kt b/app/src/test/kotlin/com/bitchat/android/noise/NoiseExternalVectorTest.kt new file mode 100644 index 00000000..70badb96 --- /dev/null +++ b/app/src/test/kotlin/com/bitchat/android/noise/NoiseExternalVectorTest.kt @@ -0,0 +1,261 @@ +package com.bitchat.android.noise + +import com.bitchat.android.noise.southernstorm.protocol.CipherState +import com.bitchat.android.noise.southernstorm.protocol.HandshakeState +import com.bitchat.android.noise.southernstorm.protocol.Noise +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertThrows +import org.junit.Test + +/** + * Cacophony/Noise-C vector for Noise_XX_25519_ChaChaPoly_SHA256. + * + * This exercises the vendored Noise state machine directly, independent of managers, Android + * storage, and generated keys. + */ +class NoiseExternalVectorTest { + private val messages = listOf( + VectorMessage( + "4c756477696720766f6e204d69736573", + "ca35def5ae56cec33dc2036731ab14896bc4c75dbb07a61f879f8e3afa4c7944" + + "4c756477696720766f6e204d69736573" + ), + VectorMessage( + "4d757272617920526f746862617264", + "95ebc60d2b1fa672c1f46a8aa265ef51bfe38e7ccb39ec5be34069f144808843" + + "81cbad1f276e038c48378ffce2b65285e08d6b68aaa3629a5a8639392490e5b9" + + "bd5269c2f1e4f488ed8831161f19b7815528f8982ffe09be9b5c412f8a0db50f" + + "8814c7194e83f23dbd8d162c9326ad" + ), + VectorMessage( + "462e20412e20486179656b", + "c7195ffacac1307ff99046f219750fc47693e23c3cb08b89c2af808b444850a8" + + "0ae475b9df0f169ae80a89be0865b57f58c9fea0d4ec82a286427402f113e4b6" + + "ae769a1d95941d49b25030" + ), + VectorMessage( + "4361726c204d656e676572", + "96763ed773f8e47bb3712f0e29b3060ffc956ffc146cee53d5e1df" + ), + VectorMessage( + "4a65616e2d426170746973746520536179", + "3e40f15f6f3a46ae446b253bf8b1d9ffb6ed9b174d272328ff91a7e2e5c79c07f5" + ), + VectorMessage( + "457567656e2042f6686d20766f6e2042617765726b", + "eb3f3515110702e047a6c9da4478b6ead94873c11c0f2d710ddb3f09fce024b3" + + "a58502ae3f" + ) + ) + + @Test + fun `Noise-C XX transcript matches every handshake and transport byte`() { + val initiator = vectorState(HandshakeState.INITIATOR) + val responder = vectorState(HandshakeState.RESPONDER) + try { + val states = listOf( + initiator to responder, + responder to initiator, + initiator to responder + ) + messages.take(3).zip(states).forEach { (message, peers) -> + assertHandshakeMessage(peers.first, peers.second, message) + } + + assertEquals(HandshakeState.SPLIT, initiator.action) + assertEquals(HandshakeState.SPLIT, responder.action) + assertArrayEquals(initiator.handshakeHash, responder.handshakeHash) + + val initiatorCiphers = initiator.split() + val responderCiphers = responder.split() + assertTransportMessage( + responderCiphers.sender, + initiatorCiphers.receiver, + messages[3] + ) + assertTransportMessage( + initiatorCiphers.sender, + responderCiphers.receiver, + messages[4] + ) + assertTransportMessage( + responderCiphers.sender, + initiatorCiphers.receiver, + messages[5] + ) + initiatorCiphers.sender.destroy() + initiatorCiphers.receiver.destroy() + responderCiphers.sender.destroy() + responderCiphers.receiver.destroy() + } finally { + initiator.destroy() + responder.destroy() + } + } + + @Test + fun `Noise state machine rejects invalid actions and a tampered handshake tag`() { + val initiator = vectorState(HandshakeState.INITIATOR) + val responder = vectorState(HandshakeState.RESPONDER) + try { + assertThrows(IllegalStateException::class.java) { initiator.start() } + assertThrows(IllegalStateException::class.java) { + responder.writeMessage(ByteArray(256), 0, null, 0, 0) + } + + assertHandshakeMessage(initiator, responder, messages[0]) + val message2 = write(responder, messages[1].payload) + val tampered = message2.copyOf() + tampered[tampered.lastIndex] = (tampered.last().toInt() xor 1).toByte() + + assertThrows(Exception::class.java) { + initiator.readMessage(tampered, 0, tampered.size, ByteArray(256), 0) + } + assertEquals(HandshakeState.FAILED, initiator.action) + } finally { + initiator.destroy() + responder.destroy() + } + } + + @Test + fun `ChaChaPoly authentication binds nonce ciphertext tag and associated data`() { + val key = ByteArray(32) { it.toByte() } + val plaintext = "associated".toByteArray() + val associatedData = "header".toByteArray() + val sender = Noise.createCipher("ChaChaPoly") + val receiver = Noise.createCipher("ChaChaPoly") + val wrongAdReceiver = Noise.createCipher("ChaChaPoly") + try { + sender.initializeKey(key, 0) + receiver.initializeKey(key, 0) + wrongAdReceiver.initializeKey(key, 0) + sender.setNonce(7) + receiver.setNonce(7) + wrongAdReceiver.setNonce(7) + val ciphertext = ByteArray(plaintext.size + sender.macLength) + val length = sender.encryptWithAd( + associatedData, + plaintext, + 0, + ciphertext, + 0, + plaintext.size + ) + + assertThrows(Exception::class.java) { + wrongAdReceiver.decryptWithAd( + "wrong".toByteArray(), + ciphertext, + 0, + ByteArray(length), + 0, + length + ) + } + val output = ByteArray(length) + val outputLength = receiver.decryptWithAd( + associatedData, + ciphertext, + 0, + output, + 0, + length + ) + assertArrayEquals(plaintext, output.copyOf(outputLength)) + } finally { + sender.destroy() + receiver.destroy() + wrongAdReceiver.destroy() + } + } + + private fun vectorState(role: Int): HandshakeState { + val state = HandshakeState(PROTOCOL, role) + val prologue = hex("4a6f686e2047616c74") + state.setPrologue(prologue, 0, prologue.size) + val staticPrivate = if (role == HandshakeState.INITIATOR) { + hex("e61ef9919cde45dd5f82166404bd08e38bceb5dfdfded0a34c8df7ed542214d1") + } else { + hex("4a3acbfdb163dec651dfa3194dece676d437029c62a408b4c5ea9114246e4893") + } + val ephemeralPrivate = if (role == HandshakeState.INITIATOR) { + hex("893e28b9dc6ca8d611ab664754b8ceb7bac5117349a4439a6b0569da977c464a") + } else { + hex("bbdb4cdbd309f1a1f2e1456967fe288cadd6f712d65dc7b7793d5e63da6b375b") + } + state.localKeyPair.setPrivateKey(staticPrivate, 0) + state.fixedEphemeralKey.setPrivateKey(ephemeralPrivate, 0) + state.start() + return state + } + + private fun assertHandshakeMessage( + writer: HandshakeState, + reader: HandshakeState, + message: VectorMessage + ) { + val actualCiphertext = write(writer, message.payload) + assertArrayEquals(message.ciphertext, actualCiphertext) + + val plaintext = ByteArray(256) + val length = reader.readMessage( + actualCiphertext, + 0, + actualCiphertext.size, + plaintext, + 0 + ) + assertArrayEquals(message.payload, plaintext.copyOf(length)) + } + + private fun write(state: HandshakeState, payload: ByteArray): ByteArray { + val output = ByteArray(512) + val length = state.writeMessage(output, 0, payload, 0, payload.size) + return output.copyOf(length) + } + + private fun assertTransportMessage( + sender: CipherState, + receiver: CipherState, + message: VectorMessage + ) { + val encrypted = ByteArray(message.payload.size + sender.macLength) + val encryptedLength = sender.encryptWithAd( + null, + message.payload, + 0, + encrypted, + 0, + message.payload.size + ) + assertArrayEquals(message.ciphertext, encrypted.copyOf(encryptedLength)) + + val decrypted = ByteArray(encryptedLength) + val decryptedLength = receiver.decryptWithAd( + null, + encrypted, + 0, + decrypted, + 0, + encryptedLength + ) + assertArrayEquals(message.payload, decrypted.copyOf(decryptedLength)) + } + + private data class VectorMessage( + private val payloadHex: String, + private val ciphertextHex: String + ) { + val payload: ByteArray get() = hex(payloadHex) + val ciphertext: ByteArray get() = hex(ciphertextHex) + } + + companion object { + private const val PROTOCOL = "Noise_XX_25519_ChaChaPoly_SHA256" + + private fun hex(value: String): ByteArray = + value.chunked(2).map { it.toInt(16).toByte() }.toByteArray() + } +} diff --git a/app/src/test/kotlin/com/bitchat/android/noise/NoiseSessionManagerIdentityBindingTest.kt b/app/src/test/kotlin/com/bitchat/android/noise/NoiseSessionManagerIdentityBindingTest.kt index 63f5e3c1..1e18fd6a 100644 --- a/app/src/test/kotlin/com/bitchat/android/noise/NoiseSessionManagerIdentityBindingTest.kt +++ b/app/src/test/kotlin/com/bitchat/android/noise/NoiseSessionManagerIdentityBindingTest.kt @@ -330,6 +330,51 @@ class NoiseSessionManagerIdentityBindingTest { assertArrayEquals(plaintext, bobManager.decrypt(ciphertext, alice.peerID)) } + @Test + fun `simultaneous handshake collision matrix has one deterministic winner`() { + val identities = listOf( + identity("e61ef9919cde45dd5f82166404bd08e38bceb5dfdfded0a34c8df7ed542214d1"), + identity("4a3acbfdb163dec651dfa3194dece676d437029c62a408b4c5ea9114246e4893"), + identity("77076d0a7318a57d3c16c17251b26645df4c2f87ebc0992ab177fba51db92c2a"), + identity("5dab087e624a8a4b79e17f8b83800ee66f3bb1292618b6fd1c2f8b27ff88e0eb") + ) + + identities.indices.forEach { leftIndex -> + ((leftIndex + 1) until identities.size).forEach { rightIndex -> + val left = identities[leftIndex] + val right = identities[rightIndex] + val leftManager = manager(left) + val rightManager = manager(right) + val leftMessage1 = leftManager.initiateHandshake(right.peerID)!! + val rightMessage1 = rightManager.initiateHandshake(left.peerID)!! + + val leftResponse = leftManager.processHandshakeMessage(right.peerID, rightMessage1) + val rightResponse = rightManager.processHandshakeMessage(left.peerID, leftMessage1) + + if (left.peerID < right.peerID) { + assertNull(leftResponse) + val message3 = leftManager.processHandshakeMessage(right.peerID, rightResponse!!)!! + assertNull(rightManager.processHandshakeMessage(left.peerID, message3)) + } else { + assertNull(rightResponse) + val message3 = rightManager.processHandshakeMessage(left.peerID, leftResponse!!)!! + assertNull(leftManager.processHandshakeMessage(right.peerID, message3)) + } + + assertTrue(leftManager.hasEstablishedSession(right.peerID)) + assertTrue(rightManager.hasEstablishedSession(left.peerID)) + val payload = "matrix-$leftIndex-$rightIndex".toByteArray() + assertArrayEquals( + payload, + rightManager.decrypt( + leftManager.encrypt(payload, right.peerID), + left.peerID + ) + ) + } + } + } + @Test fun `peer ID derivation rejects malformed keys and non-wire claims`() { val peer = identity() @@ -381,4 +426,19 @@ class NoiseSessionManagerIdentityBindingTest { dh.destroy() } } + + private fun identity(privateKeyHex: String): TestIdentity { + val privateKey = privateKeyHex.chunked(2) + .map { it.toInt(16).toByte() } + .toByteArray() + val dh = Noise.createDH("25519") + return try { + dh.setPrivateKey(privateKey, 0) + val publicKey = ByteArray(32) + dh.getPublicKey(publicKey, 0) + TestIdentity(privateKey, publicKey, NoisePeerIdentity.derivePeerID(publicKey)!!) + } finally { + dh.destroy() + } + } } diff --git a/app/src/test/kotlin/com/bitchat/android/nostr/NostrRelayManagerLifecycleSmokeTest.kt b/app/src/test/kotlin/com/bitchat/android/nostr/NostrRelayManagerLifecycleSmokeTest.kt new file mode 100644 index 00000000..7bb4aa30 --- /dev/null +++ b/app/src/test/kotlin/com/bitchat/android/nostr/NostrRelayManagerLifecycleSmokeTest.kt @@ -0,0 +1,51 @@ +package com.bitchat.android.nostr + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +@RunWith(RobolectricTestRunner::class) +class NostrRelayManagerLifecycleSmokeTest { + @Test + fun `disconnected manager maintains subscription and empty publish invariants locally`() { + val manager = NostrRelayManager.shared + manager.disconnect() + manager.clearAllSubscriptions() + + val id = manager.subscribe( + filter = NostrFilter(kinds = listOf(NostrKind.TEXT_NOTE)), + id = "local-contract", + handler = {}, + targetRelayUrls = emptyList() + ) + + assertEquals("local-contract", id) + assertEquals(1, manager.getActiveSubscriptionCount()) + assertTrue(manager.getActiveSubscriptions().containsKey(id)) + assertTrue(manager.validateSubscriptionConsistency().isConsistent) + manager.sendEvent(signedEvent(), relayUrls = emptyList()) + manager.retryConnection("wss://not-configured.example") + + manager.unsubscribe(id) + assertEquals(0, manager.getActiveSubscriptionCount()) + assertFalse(manager.isConnected.value) + assertTrue(manager.getRelayStatuses().none { it.isConnected }) + + manager.disconnect() + assertFalse(manager.isConnected.value) + } + + private fun signedEvent(): NostrEvent { + val privateKey = "0".repeat(63) + "1" + return NostrEvent( + pubkey = NostrCrypto.derivePublicKey(privateKey), + createdAt = 1, + kind = NostrKind.TEXT_NOTE, + tags = emptyList(), + content = "local" + ).sign(privateKey) + } +} diff --git a/app/src/test/kotlin/com/bitchat/android/onboarding/SystemStateManagerContractTest.kt b/app/src/test/kotlin/com/bitchat/android/onboarding/SystemStateManagerContractTest.kt new file mode 100644 index 00000000..a3f0341f --- /dev/null +++ b/app/src/test/kotlin/com/bitchat/android/onboarding/SystemStateManagerContractTest.kt @@ -0,0 +1,87 @@ +package com.bitchat.android.onboarding + +import android.app.Application +import android.bluetooth.BluetoothManager +import android.content.Context +import android.location.LocationManager +import androidx.activity.ComponentActivity +import androidx.test.core.app.ApplicationProvider +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.Robolectric +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows.shadowOf +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [35], application = Application::class) +class SystemStateManagerContractTest { + @Test + fun `Bluetooth disabled and enabled states are observable without throwing`() { + val app = ApplicationProvider.getApplicationContext() + val controller = Robolectric.buildActivity(ComponentActivity::class.java).create() + val adapter = app.getSystemService(BluetoothManager::class.java).adapter + val manager = BluetoothStatusManager( + activity = controller.get(), + context = app, + onBluetoothEnabled = {}, + onBluetoothDisabled = {} + ) + + shadowOf(adapter).setEnabled(false) + assertEquals(BluetoothStatus.DISABLED, manager.checkBluetoothStatus()) + shadowOf(adapter).setEnabled(true) + assertEquals(BluetoothStatus.ENABLED, manager.checkBluetoothStatus()) + controller.destroy() + } + + @Test + fun `location disabled and enabled states are observable and receiver is cleaned up`() { + val app = ApplicationProvider.getApplicationContext() + val controller = Robolectric.buildActivity(ComponentActivity::class.java).create() + val locationManager = app.getSystemService(Context.LOCATION_SERVICE) as LocationManager + val manager = LocationStatusManager( + activity = controller.get(), + context = app, + onLocationEnabled = {}, + onLocationDisabled = {} + ) + + shadowOf(locationManager).setLocationEnabled(false) + assertEquals(LocationStatus.DISABLED, manager.checkLocationStatus()) + shadowOf(locationManager).setLocationEnabled(true) + assertEquals(LocationStatus.ENABLED, manager.checkLocationStatus()) + + manager.cleanup() + manager.cleanup() + controller.destroy() + } + + @Test + fun `location status routing and recovery messages remain exact`() { + val app = ApplicationProvider.getApplicationContext() + val controller = Robolectric.buildActivity(ComponentActivity::class.java).create() + var enabled = 0 + val disabled = mutableListOf() + val manager = LocationStatusManager( + activity = controller.get(), + context = app, + onLocationEnabled = { enabled++ }, + onLocationDisabled = disabled::add + ) + + manager.handleLocationStatus(LocationStatus.ENABLED) + manager.handleLocationStatus(LocationStatus.NOT_AVAILABLE) + + assertEquals(1, enabled) + assertEquals( + listOf("Location services are not available on this device."), + disabled + ) + assertTrue(manager.getStatusMessage(LocationStatus.DISABLED).contains("Bluetooth scanning")) + manager.cleanup() + controller.destroy() + } +} diff --git a/app/src/test/kotlin/com/bitchat/android/ui/MeshDelegateHandlerStateContractTest.kt b/app/src/test/kotlin/com/bitchat/android/ui/MeshDelegateHandlerStateContractTest.kt new file mode 100644 index 00000000..63ba28ef --- /dev/null +++ b/app/src/test/kotlin/com/bitchat/android/ui/MeshDelegateHandlerStateContractTest.kt @@ -0,0 +1,141 @@ +package com.bitchat.android.ui + +import com.bitchat.android.mesh.MeshService +import com.bitchat.android.model.BitchatMessage +import com.bitchat.android.model.DeliveryStatus +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.TestScope +import kotlinx.coroutines.test.UnconfinedTestDispatcher +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.times +import org.mockito.kotlin.verify +import java.util.Date +import java.util.concurrent.atomic.AtomicInteger + +@OptIn(ExperimentalCoroutinesApi::class) +class MeshDelegateHandlerStateContractTest { + private lateinit var state: ChatState + private lateinit var messages: MessageManager + private lateinit var channels: ChannelManager + private lateinit var privateChats: PrivateChatManager + private lateinit var notifications: NotificationManager + private lateinit var mesh: MeshService + private lateinit var handler: MeshDelegateHandler + private lateinit var haptics: AtomicInteger + + @Before + fun setUp() { + val scope = TestScope(UnconfinedTestDispatcher()) + state = ChatState(scope) + state.setNickname("Résumé") + messages = MessageManager(state) + channels = mock() + privateChats = mock() + notifications = mock() + mesh = mock() + haptics = AtomicInteger() + handler = MeshDelegateHandler( + state = state, + messageManager = messages, + channelManager = channels, + privateChatManager = privateChats, + notificationManager = notifications, + coroutineScope = scope, + onHapticFeedback = { haptics.incrementAndGet() }, + getMyPeerID = { "self" }, + getMeshService = { mesh } + ) + } + + @Test + fun `peer arrival deduplicates list and final departure restores disconnected state`() { + handler.didUpdatePeerList(listOf("peer-a", "peer-a", "peer-b")) + + assertEquals(listOf("peer-a", "peer-b"), state.connectedPeers.value) + assertTrue(state.isConnected.value) + verify(notifications).showActiveUserNotification(listOf("peer-a", "peer-b")) + verify(channels).cleanupDisconnectedMembers(listOf("peer-a", "peer-b"), "self") + + handler.didUpdatePeerList(emptyList()) + + assertTrue(state.connectedPeers.value.isEmpty()) + assertFalse(state.isConnected.value) + verify(notifications).showActiveUserNotification(emptyList()) + } + + @Test + fun `delivery and read callbacks advance visible status monotonically`() { + val outgoing = message( + id = "outgoing", + sender = "me", + deliveryStatus = DeliveryStatus.Sending + ) + state.setMessages(listOf(outgoing)) + + handler.didReceiveDeliveryAck("outgoing", "peer-a") + assertTrue(state.messages.value.single().deliveryStatus is DeliveryStatus.Delivered) + + handler.didReceiveReadReceipt("outgoing", "peer-a") + assertTrue(state.messages.value.single().deliveryStatus is DeliveryStatus.Read) + + handler.didReceiveDeliveryAck("outgoing", "peer-a") + assertTrue(state.messages.value.single().deliveryStatus is DeliveryStatus.Read) + } + + @Test + fun `unicode mention notifies once and duplicate transport delivery is suppressed`() { + val incoming = message( + id = "incoming", + sender = "alice", + content = "hello @résumé", + senderPeerID = "peer-a" + ) + + handler.didReceiveMessage(incoming) + handler.didReceiveMessage(incoming) + + assertEquals(1, haptics.get()) + verify(notifications, times(1)).showMeshMentionNotification( + senderNickname = eq("alice"), + messageContent = eq("hello @résumé"), + senderPeerID = eq("peer-a") + ) + } + + @Test + fun `channel inbound increments unread only when conversation is not focused`() { + state.setJoinedChannels(setOf("#room")) + val incoming = message(id = "channel-1", channel = "#room") + + handler.didReceiveMessage(incoming) + assertEquals(1, state.unreadChannelMessages.value["#room"]) + + state.setCurrentChannel("#room") + handler.didReceiveMessage(incoming.copy(id = "channel-2")) + assertEquals(1, state.unreadChannelMessages.value["#room"]) + } + + private fun message( + id: String, + sender: String = "alice", + content: String = id, + senderPeerID: String? = null, + channel: String? = null, + deliveryStatus: DeliveryStatus? = null + ) = BitchatMessage( + id = id, + sender = sender, + content = content, + timestamp = Date(1), + senderPeerID = senderPeerID, + channel = channel, + deliveryStatus = deliveryStatus + ) +} diff --git a/app/src/test/kotlin/com/bitchat/android/wifi-aware/SyncedSocketContractTest.kt b/app/src/test/kotlin/com/bitchat/android/wifi-aware/SyncedSocketContractTest.kt new file mode 100644 index 00000000..ed5cec9f --- /dev/null +++ b/app/src/test/kotlin/com/bitchat/android/wifi-aware/SyncedSocketContractTest.kt @@ -0,0 +1,151 @@ +package com.bitchat.android.wifiaware + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Test +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.mock +import org.mockito.kotlin.verify +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.io.DataInputStream +import java.io.DataOutputStream +import java.io.FilterInputStream +import java.io.IOException +import java.io.InputStream +import java.io.OutputStream +import java.net.Socket +import java.util.Collections + +class SyncedSocketContractTest { + @Test + fun `write emits big-endian length payload and empty keepalive frames`() { + val output = ByteArrayOutputStream() + val raw = socket(input = ByteArrayInputStream(byteArrayOf()), output = output) + val synced = SyncedSocket(raw, readTimeoutMs = 1_234) + + synced.write(byteArrayOf(1, 2, 3)) + synced.write(ByteArray(0)) + + assertArrayEquals( + byteArrayOf(0, 0, 0, 3, 1, 2, 3, 0, 0, 0, 0), + output.toByteArray() + ) + verify(raw).soTimeout = 1_234 + } + + @Test + fun `readFully reconstructs one-byte partial reads and keepalives`() { + val wire = framed(byteArrayOf(1, 2, 3, 4)) + framed(ByteArray(0)) + val partial = object : FilterInputStream(ByteArrayInputStream(wire)) { + override fun read(buffer: ByteArray, offset: Int, length: Int): Int = + super.read(buffer, offset, minOf(1, length)) + } + val synced = SyncedSocket(socket(partial, ByteArrayOutputStream())) + + assertArrayEquals(byteArrayOf(1, 2, 3, 4), synced.read()) + assertArrayEquals(ByteArray(0), synced.read()) + assertNull(synced.read()) + } + + @Test + fun `EOF truncated invalid and oversized frames fail closed`() { + val cases = listOf( + ByteArray(0), + byteArrayOf(0, 0), + byteArrayOf(0, 0, 0, 4, 1, 2), + intPrefix(-1), + intPrefix(65_537) + ) + + cases.forEach { wire -> + val synced = SyncedSocket( + socket(ByteArrayInputStream(wire), ByteArrayOutputStream()) + ) + assertNull(synced.read()) + } + } + + @Test + fun `write exceptions propagate and do not create a partial success`() { + val failingOutput = object : OutputStream() { + override fun write(value: Int) { + throw IOException("scripted write failure") + } + } + val synced = SyncedSocket(socket(ByteArrayInputStream(byteArrayOf()), failingOutput)) + + assertThrows(IOException::class.java) { + synced.write(byteArrayOf(1)) + } + } + + @Test + fun `concurrent writers produce complete non-interleaved frames`() { + val output = ByteArrayOutputStream() + val synced = SyncedSocket(socket(ByteArrayInputStream(byteArrayOf()), output)) + val payloads = (0 until 16).map { index -> + ByteArray(index + 1) { index.toByte() } + } + val failures = Collections.synchronizedList(mutableListOf()) + val threads = payloads.map { payload -> + Thread { + runCatching { synced.write(payload) } + .exceptionOrNull() + ?.let(failures::add) + }.also(Thread::start) + } + threads.forEach { thread -> + thread.join(2_000) + assertFalse("Writer thread did not complete", thread.isAlive) + } + assertTrue(failures.isEmpty()) + + val input = DataInputStream(ByteArrayInputStream(output.toByteArray())) + val decoded = mutableListOf() + while (input.available() > 0) { + val length = input.readInt() + decoded += ByteArray(length).also(input::readFully) + } + assertEquals( + payloads.map(ByteArray::toList).toSet(), + decoded.map(ByteArray::toList).toSet() + ) + } + + @Test + fun `close and raw socket status are exposed`() { + val raw = socket(ByteArrayInputStream(byteArrayOf()), ByteArrayOutputStream()) + org.mockito.kotlin.whenever(raw.isClosed).thenReturn(false, true) + org.mockito.kotlin.whenever(raw.isConnected).thenReturn(true) + val synced = SyncedSocket(raw) + + assertFalse(synced.isClosed()) + assertTrue(synced.isConnected()) + synced.close() + verify(raw).close() + assertTrue(synced.isClosed()) + } + + private fun socket(input: InputStream, output: OutputStream): Socket = mock { + on { getInputStream() } doReturn input + on { getOutputStream() } doReturn output + } + + private fun framed(payload: ByteArray): ByteArray = + ByteArrayOutputStream().also { output -> + DataOutputStream(output).use { data -> + data.writeInt(payload.size) + data.write(payload) + } + }.toByteArray() + + private fun intPrefix(value: Int): ByteArray = + ByteArrayOutputStream().also { output -> + DataOutputStream(output).use { it.writeInt(value) } + }.toByteArray() +} diff --git a/build.gradle.kts b/build.gradle.kts index 44c5199e..5aac5518 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -10,3 +10,9 @@ tasks.whenTaskAdded { enabled = false } } + +tasks.register("clientRewriteContractTest") { + group = "verification" + description = "Runs the complete compatibility gate for a from-scratch client rewrite." + dependsOn(":app:testDebugUnitTest") +} diff --git a/docs/client-rewrite-contracts.md b/docs/client-rewrite-contracts.md new file mode 100644 index 00000000..0337ccf9 --- /dev/null +++ b/docs/client-rewrite-contracts.md @@ -0,0 +1,63 @@ +# Client rewrite compatibility contracts + +This document defines the behavior a from-scratch BitChat client must preserve. +The executable source of truth is the JVM test suite under +`app/src/test/**/contracts`, together with the pre-existing protocol, security, +mesh, and state tests. + +The remaining implementation work and milestone progress are tracked in +[test-implementation-plan.md](test-implementation-plan.md). + +## Required contract layers + +| Layer | Compatibility promise | Primary tests | +|---|---|---| +| Outer mesh packet | v1/v2 header widths, big-endian fields, flags, section order, route placement, signature placement, padding, compression, signing bytes | `BinaryProtocolTest`, `ClientRewriteWireContractTest` | +| Chat payload | Flag bits, millisecond timestamp, UTF-8 byte lengths, encrypted-content substitution, optional-field order | `ClientRewriteWireContractTest` | +| Inner payloads | Noise type bytes, private-message TLVs, peer-state TLVs, file-transfer TLVs, fragment header, sync request TLVs | `ClientRewriteWireContractTest`, `AuthenticatedPeerStateTest`, `PrivateMediaTransferPreparerTest`, `FragmentManagerTest` | +| Identity/security | Announcement extensions, capability bitfield endianness, Noise static-key binding, handshake identity binding, signatures | `IdentityAnnouncementTest`, `NoiseSessionManagerIdentityBindingTest`, `ClientRewritePrimitiveContractTest` | +| Sync/routing | Stable packet IDs, GCS bitstream, replay collapse, TTL handling, relay choice, confirmed graph edges | `ClientRewritePrimitiveContractTest`, `GCSFilterTest`, `PacketRelayManagerTest`, `MeshGraphServiceTest`, `TransportBridgeServiceTest` | +| Nostr | Bech32, secp256k1 key derivation, NIP-01 event IDs/signatures, NIP-44 authenticated encryption, NIP-13 PoW, authenticated NIP-17 seals | `ClientRewriteNostrContractTest`, `NostrProtocolTest` | +| Application state | Peer unions, canonical private conversations, chronological history, delivery/read behavior, media migration policy | `AppStateStoreTest`, `PrivateChatManagerTest`, `MediaSendingManagerMigrationTest` | + +## Golden-vector policy + +Golden vectors compare literal externally visible bytes or hashes. Do not update +them merely because an implementation changed. Update a vector only when the +wire protocol is intentionally versioned and interoperating clients are updated +together. + +Round-trip tests remain useful but are not sufficient on their own: an encoder +and decoder can share the same defect. Each critical wire format therefore has +at least one literal vector. + +## Rewrite acceptance gate + +From a configured Android development environment, run: + +```sh +./gradlew clientRewriteContractTest +``` + +The task runs the new golden vectors and the complete existing unit suite. A +rewrite is compatible only when this gate passes. Tests should be ported +unchanged when package boundaries change; adapter façades are preferable to +weakening assertions. + +## Device-only acceptance + +Local JVM tests cannot prove Android radio and lifecycle behavior. Before +shipping a rewrite, run the following on at least two physical devices: + +1. BLE discovery, connection, disconnect, reconnect, and multi-hop relay. +2. Runtime permission denial/retry for Bluetooth, location, notifications, and + microphone. +3. Foreground-service survival with the screen off and after process recreation. +4. Cross-client Android/iOS exchange for announce, public/private text, delivery + and read receipts, image/audio/file transfer, sync replay, and Nostr fallback. +5. Corrupt, duplicated, reordered, delayed, and partially delivered fragments. +6. Identity rotation, verification continuity, downgrade rejection, and recovery + after stale Noise sessions. + +Those scenarios belong in instrumented tests or a two-device interoperability +harness; they must not be represented as passing JVM mocks. diff --git a/docs/device-transport-test-matrix.md b/docs/device-transport-test-matrix.md new file mode 100644 index 00000000..8ada4018 --- /dev/null +++ b/docs/device-transport-test-matrix.md @@ -0,0 +1,113 @@ +# Physical transport validation matrix + +This is the device-only companion to the deterministic Milestone 4 transport +suite. It validates that the fake adapters, Robolectric behavior, and pure +state machines match Android framework behavior. It is also consumed by the +Milestone 10 release gate. + +## Required device set + +- Two physical Android devices from different manufacturers. +- At least one Android 13+ device for `NEARBY_WIFI_DEVICES`. +- At least one device that supports Wi-Fi Aware. +- Bluetooth LE central and peripheral support on both devices. +- A build from the exact commit under test installed on both devices. +- Clean app data before the first run; retain a second run for restart tests. + +Record device models, API levels, build commit, negotiated MTUs, and timestamps +in the release artifact. Do not record user names, device serials, Bluetooth +addresses, IP addresses, peer IDs, message contents, or other identifying +values. + +## BLE discovery and recovery + +- [ ] Start both clients and confirm each begins scanning and advertising. +- [ ] Stop and restart the foreground service; confirm exactly one scanner and + advertiser generation remains active. +- [ ] Toggle Bluetooth off during scanning, then on; confirm scanning, + advertising, announcements, and peer discovery recover without process + restart. +- [ ] Disable and re-enable the BLE debug transport; confirm the same service + instance can recover without duplicate callbacks. +- [ ] Rotate the observed BLE address by restarting advertising; confirm the + canonical peer remains singular. +- [ ] Trigger a transient scan failure or Android Bluetooth process restart; + confirm bounded retry and watchdog recovery. +- [ ] Confirm permission denial reports unavailable state without a crash, + prompt loop, or active radio work. + +## GATT setup and teardown + +- [ ] Connect in both directions simultaneously and confirm one canonical link + survives. +- [ ] Record the negotiated MTU and repeat at 23, 247, and 517 where the device + or test peripheral allows it. +- [ ] Remove the service, characteristic, or CCCD in a test peripheral and + confirm setup fails closed. +- [ ] Reject notification registration and descriptor writes; confirm the peer + is never published ready. +- [ ] Disconnect during MTU negotiation, service discovery, subscription, + client write, and server notification; confirm no stale ready callback. +- [ ] Leave setup incomplete for more than 30 seconds; confirm timeout and + resource closure. +- [ ] Connect beyond configured client, server, and total limits; confirm + deterministic oldest-link eviction. + +## Packet delivery and fragmentation + +- [ ] Send directed and broadcast packets over client and server roles. +- [ ] Saturate each link faster than radio completion callbacks; confirm one + outstanding operation, bounded backpressure, and no reordered frames. +- [ ] Inject a failed `onCharacteristicWrite` and `onNotificationSent`; confirm + queued work is discarded and the failed generation is cleaned up. +- [ ] Transfer payloads immediately below and above the fragmentation boundary. +- [ ] Transfer a maximum admitted private-media payload at negotiated MTU 517. +- [ ] At MTU 247 and 23, confirm oversized frames are rejected rather than + partially sent. Adaptive per-link fragmentation remains tracked as + `TDB-023`. +- [ ] Disconnect and reconnect halfway through a fragmented transfer; confirm + incomplete state expires and a fresh transfer can finish. +- [ ] Cancel a queued transfer and stop the service during another; confirm no + later fragments or progress callbacks. + +## Wi-Fi Aware + +- [ ] Confirm unsupported hardware and temporarily unavailable radio states are + distinct. +- [ ] Deny and grant `NEARBY_WIFI_DEVICES`; confirm publish/subscribe work only + after grant. +- [ ] Start and stop publish and subscribe sessions repeatedly; confirm no + duplicate discovery callbacks. +- [ ] Authenticate a provisional socket and promote it to the canonical peer. +- [ ] Replace a socket while authentication is in flight; confirm the stale + socket cannot promote or deliver. +- [ ] Toggle Wi-Fi, location, and airplane mode; confirm rediscovery and bounded + reconnect after availability returns. +- [ ] Stop the service with active sockets, server sockets, and network + callbacks; confirm all are closed or unregistered. + +## Unified transport and failover + +- [ ] Connect the same peer over BLE and Wi-Fi Aware; confirm one peer-list row. +- [ ] Send with both transports active; confirm the preferred transport is used. +- [ ] Drop the preferred transport during a transfer and confirm defined + failover behavior without duplicate application delivery. +- [ ] Relay between transports and confirm TTL decreases once per hop. +- [ ] Reflect a bridged packet back over the other transport; confirm loop and + duplicate suppression. +- [ ] Stop the foreground service; confirm scans, advertisements, sessions, + sockets, operation queues, transfer jobs, and callbacks all terminate. + +## Evidence template + +| Field | Value | +|---|---| +| Commit | | +| Device/API classes | | +| BLE central/peripheral | Pass / Fail | +| MTU cases | Pass / Fail / Unsupported | +| BLE recovery | Pass / Fail | +| Wi-Fi Aware lifecycle | Pass / Fail / Unsupported | +| Cross-transport failover | Pass / Fail | +| Shutdown leak check | Pass / Fail | +| Bugs filed | | diff --git a/docs/release-gate-runbook.md b/docs/release-gate-runbook.md new file mode 100644 index 00000000..414bfbae --- /dev/null +++ b/docs/release-gate-runbook.md @@ -0,0 +1,244 @@ +# Physical-device and cross-client release gate + +This runbook turns Milestone 10 into a repeatable release procedure. The gate +uses a host-side CLI and USB/ADB as its control channel, so control traffic +never shares BLE, Wi-Fi Aware, Nostr, or Tor with the system under test. + +The gate cannot pass without the required physical devices and counterpart +clients. A pending or blocked result is useful diagnostic evidence, but it is +not release approval. + +## Safety and privacy rules + +- Use only disposable lab app data, identities, nicknames, messages, and files. +- Never use a personal Nostr account or a production relay. +- Do not put device serials, UDIDs, Bluetooth/MAC/IP addresses, peer IDs, + usernames, email addresses, local home paths, or message contents in a + result, trace, filename, issue, commit, or release artifact. +- Device selectors may be supplied to ADB commands as ephemeral inputs. The + tooling emits only logical aliases such as `android-current`. +- Models, manufacturer classes, Android API levels, negotiated MTU classes, + client versions, commit hashes, aggregate counts, durations, and stable + failure reason codes are allowed. +- Do not archive raw logcat. Convert observations to the structured, + privacy-checked trace format, and keep any raw diagnostic capture local until + it has been reviewed and sanitized. + +The validator rejects known identifying fields and values before a passing +bundle can be created. + +## Required lab + +Prepare: + +- At least three physical Android devices for three-hop relay testing. +- At least two Android API levels and two manufacturer classes. +- BLE central and peripheral support on every Android device. +- At least one Android 13+ device with Wi-Fi Aware. +- One physical device running the current iOS client. +- The last supported Android client. +- The release-candidate APK built from one exact full Git commit. +- A local, disposable Nostr relay/Tor fixture with production network access + blocked. + +One physical handset may be reused for the legacy-client phase after the +current-client evidence for that slot is complete, but the matrix must keep the +logical aliases and installed client versions unambiguous. + +## 1. Verify the deterministic gate + +From the repository root: + +```sh +./gradlew clientRewriteContractTest checkChangedLineCoverage lintDebug +python3 tools/release_gate/release_gate.py validate-manifest +``` + +Do not begin device work from a dirty tree or a build whose deterministic gate +does not pass. + +## 2. Create the device matrix + +Copy `tools/release_gate/device-matrix.example.json` to an ignored working +directory under `release-gate-results/`. Replace every template value and set +both current-Android commit fields to the exact full commit under test. + +Probe Android capabilities without storing the ADB selector: + +```sh +python3 tools/release_gate/android_lab.py probe \ + --serial "$BITCHAT_ADB_SELECTOR" \ + --alias android-current +``` + +Copy only the returned logical metadata into the matrix. Validate it: + +```sh +python3 tools/release_gate/release_gate.py validate-matrix \ + --matrix release-gate-results/device-matrix.json \ + --commit "$BITCHAT_RELEASE_COMMIT" +``` + +The matrix validator enforces physical devices, three Android participants, two +API levels, two manufacturer classes, Wi-Fi Aware, BLE roles, iOS, and explicit +current/legacy client versions. + +## 3. Initialize disposable fixtures + +```sh +python3 tools/release_gate/release_gate.py init \ + --matrix release-gate-results/device-matrix.json \ + --commit "$BITCHAT_RELEASE_COMMIT" \ + --run-id rc-lab-01 \ + --output release-gate-results/rc-lab-01 +``` + +Initialization pins the scenario and fixture manifests, creates every scenario +as `pending`, and generates deterministic: + +- zero-byte and small files; +- a Unicode-named medium file; +- sparse exact-maximum and oversized boundary files. + +The fixture manifest records size and SHA-256. The final archive contains the +manifest, not the large fixture bodies. + +Clear only the disposable app data on each selected lab device: + +```sh +python3 tools/release_gate/android_lab.py prepare \ + --serial "$BITCHAT_ADB_SELECTOR" \ + --confirm-disposable-app-data +``` + +This stops the app and runs package-data cleanup. The explicit confirmation is +required because the operation is destructive to that app's local data. + +## 4. Execute scenarios + +The canonical scenario list is +`tools/release_gate/scenarios.json`. It contains 27 mandatory scenarios: + +- the complete physical transport matrix; +- Android API/manufacturer/permission/background coverage; +- 11 Android-to-Android workflows; +- 8 cross-client/backward-compatibility workflows; +- 6 background and endurance workflows. + +For each scenario: + +1. Confirm the listed participants and capabilities. +2. Perform the corresponding steps in + [device-transport-test-matrix.md](device-transport-test-matrix.md) and the + Milestone 10 checklist. +3. Record connection, lifecycle, transport, receipt, resource, and terminal + state as aggregate evidence. +4. Append at least one structured trace event. +5. Mark the scenario `pass`, `fail`, `blocked`, or `unsupported`. + +Record evidence with the exact keys declared by the scenario: + +```sh +python3 tools/release_gate/release_gate.py record \ + --run release-gate-results/rc-lab-01 \ + --scenario A2A-001 \ + --status pass \ + --evidence connection-transitions=4 \ + --evidence packet-correlation-count=6 \ + --evidence failure-reasons=none +``` + +Append a privacy-safe trace event: + +```sh +python3 tools/release_gate/release_gate.py trace \ + --run release-gate-results/rc-lab-01 \ + --scenario A2A-001 \ + --source android-current \ + --event reconnect-terminal \ + --outcome pass \ + --metric reconnect-count=1 \ + --metric duplicate-delivery-count=0 +``` + +Capture resource snapshots during endurance work: + +```sh +python3 tools/release_gate/android_lab.py snapshot \ + --serial "$BITCHAT_ADB_SELECTOR" \ + --alias android-current \ + --run release-gate-results/rc-lab-01 \ + --scenario END-003 +``` + +Use run-local sequential correlation labels while observing packets; archive +only aggregate correlation counts. Record failures with a stable reason code, +file a regression issue, and preserve the incomplete artifact. + +## 5. Endurance requirements + +- `END-001` requires at least 240 minutes. +- `END-002` requires at least 50 large-transfer/cancellation cycles. +- Sample memory, threads, file descriptors, wake locks, connection counts, and + late callbacks at consistent intervals. +- A passing result requires bounded resource behavior and a clean terminal + state; merely completing the time window is insufficient. + +The validator rejects shorter durations and cycle counts. + +## 6. Inspect progress and validate + +During a run: + +```sh +python3 tools/release_gate/release_gate.py validate \ + --run release-gate-results/rc-lab-01 \ + --allow-incomplete + +python3 tools/release_gate/release_gate.py summary \ + --run release-gate-results/rc-lab-01 +``` + +The release validator, without `--allow-incomplete`, requires: + +- every scenario to be `pass`; +- every declared evidence field; +- at least one structured trace per scenario; +- the pinned scenario and fixture manifests; +- the exact client commit and complete device matrix; +- endurance minimums; +- a completion timestamp; +- no detected identifying fields or values. + +`unsupported`, `blocked`, and `pending` never satisfy release approval. + +## 7. Archive release approval + +After the complete validator passes: + +```sh +python3 tools/release_gate/release_gate.py bundle \ + --run release-gate-results/rc-lab-01 \ + --output release-gate-results/rc-lab-01.zip +``` + +The deterministic archive contains the scenario manifest, device/client matrix, +results, structured trace, fixture manifest, Markdown summary, and +`SHA256SUMS`. Attach it to the release approval record without renaming fields +or adding raw diagnostics. + +Finally, clean the disposable app data with the same confirmed `cleanup` +command and stop the local relay/Tor fixture. + +## Failure handling + +- `fail`: behavior violated a contract. Record a stable reason code, file a bug, + add a deterministic regression where possible, fix it, and rerun the affected + scenario plus dependent scenarios. +- `blocked`: required lab infrastructure or counterpart client was unavailable. + Preserve the artifact and do not approve release. +- `unsupported`: the selected device lacks a capability. Because the defined + matrix requires Wi-Fi Aware, replace the device or matrix; unsupported does + not waive a mandatory scenario. +- A flaky result is a failure until its cause is understood. Never average + retries into a pass. diff --git a/docs/test-implementation-plan.md b/docs/test-implementation-plan.md new file mode 100644 index 00000000..e64b633b --- /dev/null +++ b/docs/test-implementation-plan.md @@ -0,0 +1,779 @@ +# Test implementation plan + +## Objective + +Build enough deterministic, adversarial, integration, and device-level coverage +that the Android client can be rewritten from scratch without silently changing +its wire behavior, security properties, delivery semantics, lifecycle behavior, +or user-visible workflows. + +The canonical compatibility requirements are documented in +[client-rewrite-contracts.md](client-rewrite-contracts.md). This plan describes +how to turn those requirements into a complete, continuously enforced test +program. + +## Status legend + +- **Complete**: acceptance criteria are met and the tests run in the rewrite gate. +- **In progress**: implementation has started but acceptance criteria are not met. +- **Not started**: no implementation work has been completed. +- **Blocked**: progress requires an external dependency, device, or decision. + +## Current progress + +| Milestone | Status | Progress | Depends on | +|---|---|---:|---| +| 0. Compatibility baseline | Complete | 100% | — | +| 1. Coverage and deterministic test infrastructure | Not started | 0% | 0 | +| 2. Adversarial protocol and parser testing | Not started | 0% | 1 | +| 3. Noise, cryptography, and identity testing | Not started | 0% | 1 | +| 4. BLE, Wi-Fi Aware, and transport lifecycle testing | Not started | 0% | 1, 3 | +| 5. Sync, routing, and store-and-forward testing | Not started | 0% | 1, 4 | +| 6. Nostr and Tor integration testing | Not started | 0% | 1, 3 | +| 7. Android lifecycle and permission testing | Not started | 0% | 1, 4 | +| 8. Persistence, migration, and recovery testing | Not started | 0% | 1, 3 | +| 9. UI, media, and accessibility testing | Not started | 0% | 1, 7, 8 | +| 10. Physical-device and cross-client release gate | Not started | 0% | 2–9 | + +Milestone completion is currently **1 of 11 milestones (9%)**. This is +milestone-based progress, not line or branch coverage. Milestone 1 will establish +measured coverage baselines and trends. + +## Test levels and execution policy + +| Level | Purpose | Expected execution | +|---|---|---| +| Pure JVM unit tests | Protocols, state machines, crypto vectors, parsing, routing, and deterministic utilities | Every pull request | +| Property and fuzz tests | Malformed inputs, boundary exploration, invariants, and crash resistance | Bounded set on every pull request; extended corpus nightly | +| Robolectric tests | Android services, lifecycle, broadcasts, permissions, persistence, and process recreation | Every pull request where stable | +| Instrumented emulator tests | Compose semantics, navigation, database/filesystem integration, and permission flows | Main branch and release candidates | +| Physical-device tests | BLE, Wi-Fi Aware, radios, background execution, and manufacturer-specific behavior | Nightly where devices are available; mandatory release gate | +| Cross-client interoperability | Android/iOS and old/new client wire compatibility | Mandatory release gate | + +## Global rules + +- [x] Keep literal golden vectors for externally visible bytes and hashes. +- [x] Run all compatibility and regression tests through + `./gradlew clientRewriteContractTest`. +- [ ] Prefer public behavior and stable adapter interfaces over implementation + details. +- [ ] Require deterministic clocks, randomness, dispatchers, storage, and + transports in tests. +- [ ] Never use production relay or internet availability as a test dependency. +- [ ] Every fixed protocol or security defect must receive a regression test. +- [ ] Every decoder must have positive, boundary, malformed, and fuzz coverage. +- [ ] Every asynchronous test must have bounded completion and must not use + arbitrary sleeps. +- [ ] Test failures must preserve seeds, inputs, and traces needed to reproduce + the failure. +- [ ] Golden vectors may change only with an intentional protocol version change + and coordinated interoperability review. + +--- + +## Milestone 0: Compatibility baseline + +**Status:** Complete +**Progress:** 100% + +### Scope + +Establish executable rewrite contracts for the most important deterministic +wire formats and reuse the existing regression suite as a single acceptance +gate. + +### Completed checklist + +- [x] Create an isolated workspace and + `codex/client-rewrite-contract-tests` branch. +- [x] Add literal v1 and v2 outer packet vectors. +- [x] Add public, private, optional-field, and encrypted message vectors. +- [x] Add private-message, Noise envelope, fragment, sync, identity, and file + transfer vectors. +- [x] Add padding, binary encoding, geohash, gossip, packet ID, GCS, and Noise + peer-ID contracts. +- [x] Add Bech32, secp256k1, NIP-01, NIP-44, and NIP-13 contracts. +- [x] Add required-prefix and representative truncated-input rejection tests. +- [x] Add explicit validation for malformed fragment IDs. +- [x] Add `clientRewriteContractTest` as the complete rewrite acceptance task. +- [x] Verify 32 new tests pass without skips. +- [x] Verify the full gate discovers 254 tests with zero failures or errors. +- [x] Document the remaining device-only acceptance requirements. + +### Acceptance criteria + +- [x] The original `main` workspace remains unchanged. +- [x] All new golden-vector tests pass. +- [x] The complete unit suite passes through one documented command. + +--- + +## Milestone 1: Coverage and deterministic test infrastructure + +**Status:** Not started +**Progress:** 0% + +### Goal + +Make coverage measurable and provide reusable deterministic seams so later +milestones test behavior without real time, radios, network access, or flaky +scheduling. + +### TODO checklist + +#### Coverage reporting + +- [ ] Add JaCoCo or Kover for JVM unit-test line and branch coverage. +- [ ] Generate XML and HTML reports from the rewrite acceptance task. +- [ ] Record the initial project-wide line and branch coverage baseline. +- [ ] Record package-level baselines for `mesh`, `noise`, `nostr`, `service`, + `services`, `sync`, `model`, `protocol`, `identity`, and `ui`. +- [ ] Publish coverage artifacts in CI. +- [ ] Add a changed-lines coverage check for new production code. +- [ ] Add non-regression thresholds without forcing low-value tests for trivial + generated or platform glue. +- [ ] Exclude generated code, Compose compiler output, Android resource classes, + and vendored cryptographic code from first-party coverage metrics. + +#### Deterministic seams + +- [ ] Introduce an injectable monotonic clock and wall clock. +- [ ] Introduce injectable secure and non-secure random-byte sources where + deterministic vectors are required. +- [ ] Introduce injectable coroutine dispatchers and test scopes. +- [ ] Introduce an in-memory key/value storage adapter for preferences. +- [ ] Introduce an in-memory file store with controllable I/O failures. +- [ ] Define a fake mesh transport that can connect, disconnect, delay, drop, + duplicate, corrupt, reorder, and fragment packets. +- [ ] Define fake BLE scanner, advertiser, GATT client, and GATT server adapters. +- [ ] Define a fake Wi-Fi Aware session/socket adapter. +- [ ] Define a fake Nostr relay transport or MockWebServer fixture. +- [ ] Provide reusable packet, identity, peer, graph, and message fixture + builders. +- [ ] Provide seed capture and reproduction helpers for randomized tests. +- [ ] Add test naming and directory conventions for unit, property, Robolectric, + instrumented, and interoperability suites. + +### Acceptance criteria + +- [ ] One command generates a repeatable coverage report. +- [ ] Two consecutive clean runs produce identical deterministic test results. +- [ ] Fake time and transport behavior require no wall-clock sleeps. +- [ ] CI publishes coverage and test-result artifacts. +- [ ] The plan's progress table is updated with measured baseline numbers. + +--- + +## Milestone 2: Adversarial protocol and parser testing + +**Status:** Not started +**Progress:** 0% + +### Goal + +Prove that all wire decoders preserve canonical behavior, reject unsafe input, +and never crash or allocate unreasonable memory for attacker-controlled data. + +### TODO checklist + +#### Outer packet protocol + +- [ ] Test every valid flag combination for v1 and v2. +- [ ] Test exact minimum and maximum payload sizes. +- [ ] Test sender and recipient IDs at 0, 1, 7, 8, 9, and oversized lengths. +- [ ] Test signatures at 0, 1, 63, 64, 65, and oversized lengths. +- [ ] Test route counts at 0, 1, 254, 255, and truncated route entries. +- [ ] Test unknown message type values remain safely representable or are + rejected according to the protocol contract. +- [ ] Test invalid versions, reserved flags, integer overflow, and unsigned + length conversion. +- [ ] Test trailing bytes and concatenated frames explicitly. +- [ ] Test padding boundaries around 256, 512, 1024, and 2048 bytes. +- [ ] Test malformed PKCS#7 tails and ambiguous unpadded frames. +- [ ] Test raw DEFLATE and zlib-header compatibility vectors. +- [ ] Test forged original-size fields, compression bombs, and truncated + compressed streams. +- [ ] Add encode/decode property tests for all valid packet shapes. +- [ ] Add a mutation corpus derived from every golden packet. + +#### Inner payloads and TLVs + +- [ ] Fuzz `BitchatMessage.fromBinaryPayload`. +- [ ] Fuzz `IdentityAnnouncement.decode`. +- [ ] Fuzz `AuthenticatedPeerState.decode`. +- [ ] Fuzz `PrivateMessagePacket.decode`. +- [ ] Fuzz `NoisePayload.decode`. +- [ ] Fuzz `BitchatFilePacket.decode`. +- [ ] Fuzz `FragmentPayload.decode`. +- [ ] Fuzz `RequestSyncPacket.decode`. +- [ ] Test missing, duplicated, reordered, unknown, and zero-length TLVs. +- [ ] Test truncated headers and values at every byte offset. +- [ ] Test UTF-8 ASCII, multi-byte, combining-mark, emoji, invalid-byte, and + maximum-byte-length cases. +- [ ] Test 255-byte one-byte-length boundaries. +- [ ] Test 65,535-byte two-byte-length boundaries. +- [ ] Test four-byte file content lengths and impossible content declarations. +- [ ] Test fragmented file content using one and multiple content TLVs. +- [ ] Define and test whether non-canonical but tolerated inputs re-encode + canonically. + +#### Fuzzing operations + +- [ ] Select a JVM-compatible property/fuzz framework. +- [ ] Add bounded pull-request fuzz runs with fixed seeds. +- [ ] Add extended randomized nightly runs. +- [ ] Store minimized failing inputs as regression fixtures. +- [ ] Assert no decoder throws for arbitrary byte arrays. +- [ ] Assert decoder runtime and allocations stay within configured bounds. + +### Acceptance criteria + +- [ ] Every externally reachable decoder has boundary and malformed-input tests. +- [ ] Every decoder has a bounded arbitrary-byte no-crash property. +- [ ] All discovered crashes or ambiguous contracts have regression fixtures. +- [ ] Extended fuzzing completes nightly and preserves reproduction seeds. + +--- + +## Milestone 3: Noise, cryptography, and identity testing + +**Status:** Not started +**Progress:** 0% + +### Goal + +Prove confidentiality, authenticity, identity binding, replay behavior, session +replacement, rekeying, and recovery across the complete secure-channel +lifecycle. + +### TODO checklist + +#### Known vectors and primitives + +- [ ] Add known Curve25519 key agreement vectors. +- [ ] Add known Ed25519 signing and verification vectors. +- [ ] Add known BIP-340 verification vectors. +- [ ] Add known HKDF and channel-key derivation vectors. +- [ ] Add external Noise XX handshake transcript vectors where compatible. +- [ ] Add deterministic channel encryption vectors with injected nonces. +- [ ] Test constant-time verification APIs where the underlying library exposes + an appropriate contract. + +#### Noise session lifecycle + +- [ ] Test initiator and responder handshakes without manager wrappers. +- [ ] Test all valid handshake state transitions. +- [ ] Test every invalid message for every handshake state. +- [ ] Test tampered handshake messages and remote static-key substitution. +- [ ] Test post-handshake encryption in both directions. +- [ ] Test empty, small, maximum, and fragmented plaintext. +- [ ] Test tampered ciphertext, nonce, tag, and associated data. +- [ ] Test replayed ciphertext. +- [ ] Test skipped, duplicated, and out-of-order transport messages. +- [ ] Test send and receive nonce progression. +- [ ] Test nonce exhaustion and counter-overflow behavior. +- [ ] Test rekey thresholds, successful rekey, failed rekey, and simultaneous + rekey. +- [ ] Test session reset and destruction zeroize or discard sensitive state as + designed. +- [ ] Test handshake timeouts and stale generation leases with fake time. +- [ ] Test simultaneous initiator tie-breaking across a larger peer matrix. +- [ ] Test process restart with and without persisted identity. + +#### Identity and downgrade protection + +- [ ] Test peer-ID derivation for valid and malformed static keys. +- [ ] Test signing-key rotation with authorized and unauthorized announcements. +- [ ] Test private-media capability pinning across restart. +- [ ] Test downgrade attempts after a capability has been pinned. +- [ ] Test corrupted, missing, partially written, and legacy identity storage. +- [ ] Test atomic clearing of identity, capability, and peer mappings. +- [ ] Test verification fingerprints remain stable for unchanged identities. +- [ ] Test identity replacement does not expose an established session before + authentication completes. + +### Acceptance criteria + +- [ ] Known vectors pass independently of Android storage and services. +- [ ] Replay, tampering, downgrade, and identity-substitution tests all fail + closed. +- [ ] All timeouts and rekey tests use fake time. +- [ ] No sensitive test fixtures contain production keys or user data. + +--- + +## Milestone 4: BLE, Wi-Fi Aware, and transport lifecycle testing + +**Status:** Not started +**Progress:** 0% + +### Goal + +Verify connection state machines and packet delivery across unreliable Android +transports without requiring real radios for the majority of cases. + +### TODO checklist + +#### BLE discovery and connection + +- [ ] Test scan start, stop, restart, and failure callbacks. +- [ ] Test advertising start, stop, restart, and failure callbacks. +- [ ] Test Bluetooth-off and Bluetooth-on recovery. +- [ ] Test duplicate scan results and rapidly changing peer addresses. +- [ ] Test connection success, rejection, timeout, and cancellation. +- [ ] Test simultaneous inbound and outbound connection races. +- [ ] Test canonical connection selection and duplicate-link teardown. +- [ ] Test service discovery failure and missing characteristics. +- [ ] Test GATT disconnect during discovery, negotiation, read, and write. +- [ ] Test reconnect backoff with fake time. +- [ ] Test maximum-connection enforcement and eviction policy. +- [ ] Test RSSI thresholds and power-mode transitions. + +#### Packet transfer + +- [ ] Test MTU negotiation at minimum, normal, and maximum values. +- [ ] Test partial writes and write callbacks delivered out of order. +- [ ] Test notification subscription and notification failure. +- [ ] Test queue backpressure and bounded memory use. +- [ ] Test fragmentation and reassembly across disconnect/reconnect. +- [ ] Test duplicate, missing, reordered, and corrupted fragments. +- [ ] Test cancellation cleans pending queues and transfer state. +- [ ] Test large file/media transfers under constrained MTU. +- [ ] Test broadcast and directed packet delivery. +- [ ] Test packet relay while one link disconnects. + +#### Wi-Fi Aware + +- [ ] Test feature unavailable and permission-denied behavior. +- [ ] Test publish/subscribe session creation and teardown. +- [ ] Test provisional link authentication and canonical promotion. +- [ ] Test socket replacement and stale-socket rejection. +- [ ] Test partial reads, writes, EOF, exceptions, and cancellation. +- [ ] Test reconnect and rediscovery. +- [ ] Test coexistence with BLE for the same peer. + +#### Unified transport behavior + +- [ ] Test peer-list union and removal across transports. +- [ ] Test preferred-transport selection. +- [ ] Test transparent failover between BLE and Wi-Fi Aware. +- [ ] Test duplicate packet suppression across transports. +- [ ] Test transport bridge TTL decrement and loop prevention. +- [ ] Test shutdown cancels all jobs, scans, advertisements, sockets, and queues. + +### Acceptance criteria + +- [ ] Transport state-machine tests run deterministically on the JVM or + Robolectric. +- [ ] Disconnect and cancellation tests leave no queued work or active jobs. +- [ ] Cross-transport duplicate delivery and loops are prevented. +- [ ] A smaller physical-device suite confirms the fake adapters match Android + behavior. + +--- + +## Milestone 5: Sync, routing, and store-and-forward testing + +**Status:** Not started +**Progress:** 0% + +### Goal + +Verify eventual delivery, bounded resource usage, correct routing, and duplicate +suppression during partitions, topology changes, and reconnects. + +### TODO checklist + +#### Packet identity and sync filters + +- [ ] Add more packet-ID vectors for every message type. +- [ ] Prove TTL, route, recipient, and signature mutations do not change sync + identity. +- [ ] Prove payload, sender, timestamp, and type mutations do change identity. +- [ ] Property-test GCS encode/decode membership. +- [ ] Test empty, singleton, maximum-capacity, duplicate, and collision-heavy + filters. +- [ ] Test false-positive behavior statistically against configured tolerances. +- [ ] Test maximum accepted filter bytes and malicious bitstreams. +- [ ] Test sync requests with unknown TLVs and future capability extensions. + +#### Store and forward + +- [ ] Test caching decisions for public, private, favorite, and offline peers. +- [ ] Test cache capacity and deterministic eviction. +- [ ] Test cache expiry with fake time. +- [ ] Test delivery acknowledgement removal. +- [ ] Test retransmission after reconnect. +- [ ] Test duplicate acknowledgements and late acknowledgements. +- [ ] Test process restart persistence policy. +- [ ] Test shutdown and cleanup under active delivery. +- [ ] Test memory bounds under repeated undeliverable messages. + +#### Routing and topology + +- [ ] Test shortest paths for disconnected, cyclic, diamond, and changing graphs. +- [ ] Test deterministic tie-breaking for equal-length routes. +- [ ] Test only confirmed edges are used. +- [ ] Test edge expiry and peer disappearance with fake time. +- [ ] Test a route invalidated between planning and send. +- [ ] Test relay TTL exhaustion at every hop. +- [ ] Test source-route loop rejection. +- [ ] Test broadcast storm suppression. +- [ ] Test delivery across mixed BLE and Wi-Fi Aware paths. +- [ ] Test graph updates while sync and relay operations run concurrently. + +### Acceptance criteria + +- [ ] Partition/reconnect scenarios eventually deliver exactly once at the + application layer. +- [ ] Cache, graph, and filter resource bounds are enforced. +- [ ] No topology or bridge scenario produces an infinite relay loop. +- [ ] All expiry behavior uses fake time. + +--- + +## Milestone 6: Nostr and Tor integration testing + +**Status:** Not started +**Progress:** 0% + +### Goal + +Verify relay communication, subscriptions, event validation, NIP-17 delivery, +and Tor-mode behavior under realistic network failures. + +### TODO checklist + +#### Relay protocol + +- [ ] Add a scripted local WebSocket relay fixture. +- [ ] Test initial connection and clean disconnect. +- [ ] Test DNS, TCP, TLS, WebSocket, and protocol failures. +- [ ] Test reconnect backoff and cancellation with fake time. +- [ ] Test relay notices, acknowledgements, end-of-stored-events, and malformed + messages. +- [ ] Test subscription creation, replacement, unsubscribe, and reconnect + restoration. +- [ ] Test duplicate, delayed, reordered, and conflicting events. +- [ ] Test multi-relay publish success, partial success, and total failure. +- [ ] Test event deduplication across relays. +- [ ] Test relay-list selection and invalid relay URLs. + +#### Event security and messaging + +- [ ] Add external NIP-01, NIP-13, NIP-17, and NIP-44 vectors. +- [ ] Test invalid event IDs and signatures are rejected before dispatch. +- [ ] Test future timestamps, stale timestamps, and integer boundaries. +- [ ] Test NIP-17 gift-wrap signer/rumor identity mismatches. +- [ ] Test malformed seals, wrong recipients, and tampered ciphertext. +- [ ] Test private-message and acknowledgement embedding/extraction. +- [ ] Test geohash note, presence, and ephemeral-event filters. +- [ ] Test nickname and teleport tags. +- [ ] Test proof-of-work policy at exact difficulty boundaries. +- [ ] Test cancellation and bounded mining iterations. + +#### Tor behavior + +- [ ] Add a fake Tor-state provider and proxy-selection tests. +- [ ] Test direct, Tor-only, and fallback modes. +- [ ] Test bootstrap delay, bootstrap failure, proxy failure, and shutdown. +- [ ] Verify Tor-only mode never silently uses a direct connection. +- [ ] Verify mode changes rebuild clients and close old connections. + +### Acceptance criteria + +- [ ] Nostr integration tests require no public relay or internet connection. +- [ ] Invalid or unauthenticated events never reach application state. +- [ ] Reconnect restores intended subscriptions without duplicate delivery. +- [ ] Tor-only policy fails closed. + +--- + +## Milestone 7: Android lifecycle and permission testing + +**Status:** Not started +**Progress:** 0% + +### Goal + +Verify the app behaves correctly under Android process, service, permission, +Bluetooth, battery, and background-execution rules. + +### TODO checklist + +#### Foreground service + +- [ ] Add Robolectric tests for service create, start, bind, unbind, and destroy. +- [ ] Test repeated start commands are idempotent. +- [ ] Test foreground notification creation and channel configuration. +- [ ] Test explicit shutdown clears transport and application state correctly. +- [ ] Test unexpected process/service recreation restores required state. +- [ ] Test task removal behavior. +- [ ] Test boot-completed handling. +- [ ] Test service start restrictions and failure reporting. +- [ ] Test all coroutines and resources are cancelled on destroy. + +#### Permissions and system state + +- [ ] Test first-run permission explanations. +- [ ] Test denial, permanent denial, and later grant. +- [ ] Test partial Bluetooth permission grants by Android version. +- [ ] Test location-disabled and Bluetooth-disabled states. +- [ ] Test notification permission denial. +- [ ] Test microphone permission denial during voice recording. +- [ ] Test background-location preferences where applicable. +- [ ] Test battery-optimization accepted, declined, and unavailable paths. +- [ ] Test configuration changes during onboarding. +- [ ] Test onboarding restoration after process recreation. + +#### Android-version matrix + +- [ ] Define minimum, target, and newest-supported API test matrix. +- [ ] Add emulator coverage for behavior changes in permissions and foreground + services. +- [ ] Add at least one low-memory/process-death scenario. +- [ ] Add manufacturer-device coverage for known BLE/background differences. + +### Acceptance criteria + +- [ ] Critical service and permission flows have Robolectric or instrumented + coverage. +- [ ] No permission denial crashes or leaves onboarding irrecoverable. +- [ ] Service recreation does not duplicate transports or lose required state. +- [ ] Required API-level matrix passes before release. + +--- + +## Milestone 8: Persistence, migration, and recovery testing + +**Status:** Not started +**Progress:** 0% + +### Goal + +Ensure identities, settings, aliases, favorites, bookmarks, messages, and +capability pins survive upgrades and fail safely when storage is incomplete or +corrupt. + +### TODO checklist + +- [ ] Inventory every persisted key, file, schema, and version marker. +- [ ] Create legacy fixtures for every supported application version. +- [ ] Test clean first launch with no persisted state. +- [ ] Test upgrade from each retained legacy fixture. +- [ ] Test unknown future fields are preserved or ignored safely. +- [ ] Test truncated, malformed, empty, and type-mismatched preference values. +- [ ] Test partial multi-key identity writes. +- [ ] Test storage write failure and rollback. +- [ ] Test concurrent readers and writers. +- [ ] Test alias merging and canonical conversation migration. +- [ ] Test chronological ordering after migration. +- [ ] Test favorite and bookmark preservation. +- [ ] Test message-retention expiry with fake time. +- [ ] Test secure identity clearing removes all linked mappings and pins. +- [ ] Test signing-key and capability rotation is atomic. +- [ ] Test backup/restore policy does not duplicate or expose sensitive identity + material. +- [ ] Test migration idempotence by running each migration twice. +- [ ] Test downgrade behavior when a newer schema has already been written. + +### Acceptance criteria + +- [ ] Every supported legacy fixture migrates deterministically. +- [ ] Failed migrations leave either the old valid state or the new valid state, + never a partial mixture. +- [ ] Security-sensitive corruption fails closed with a recoverable user path. +- [ ] Migration and retention behavior uses deterministic storage and time. + +--- + +## Milestone 9: UI, media, and accessibility testing + +**Status:** Not started +**Progress:** 0% + +### Goal + +Protect user-visible behavior and media workflows while keeping most assertions +at ViewModel/state boundaries and reserving Compose instrumentation for genuine +interaction and rendering contracts. + +### TODO checklist + +#### ViewModels and application state + +- [ ] Restore or replace the currently skipped command-processor tests. +- [ ] Restore or replace the currently skipped notification tests. +- [ ] Test public/private/channel conversation switching. +- [ ] Test optimistic send, success, failure, retry, and cancellation. +- [ ] Test delivery and read-receipt transitions. +- [ ] Test peer arrival, departure, alias change, and identity rotation. +- [ ] Test state restoration after configuration change and process recreation. +- [ ] Test concurrent inbound messages while changing conversations. +- [ ] Test error messages for permission, transport, storage, and crypto failures. + +#### Compose UI + +- [ ] Add semantics tests for critical chat actions. +- [ ] Test onboarding navigation and recoverability. +- [ ] Test empty, loading, connected, disconnected, and error states. +- [ ] Test long nicknames, messages, channels, and localized text. +- [ ] Test dynamic font sizes and display scaling. +- [ ] Test light, dark, and supported theme variants. +- [ ] Add screenshot tests only for stable high-value layouts. +- [ ] Test keyboard, focus, back navigation, and bottom-sheet behavior. +- [ ] Test screen-reader labels, traversal order, and minimum touch targets. +- [ ] Test reduced-motion behavior where animations are nonessential. + +#### Files, images, and voice + +- [ ] Test zero-byte, small, maximum-size, and oversized files. +- [ ] Test unsupported and misleading MIME types. +- [ ] Test missing filenames and Unicode filenames. +- [ ] Test file read/write failures and insufficient storage. +- [ ] Test image decode failures, orientation metadata, and large-image memory + limits. +- [ ] Test voice-recording start, pause/stop, cancellation, and microphone loss. +- [ ] Test corrupt and unsupported audio playback. +- [ ] Test waveform generation boundaries. +- [ ] Test interrupted private-media preparation and commit rollback. +- [ ] Test cleanup of temporary files after success, failure, and cancellation. + +### Acceptance criteria + +- [ ] Critical user journeys pass through state-level tests. +- [ ] A focused Compose suite protects navigation, semantics, and accessibility. +- [ ] Media failures are visible, recoverable, and leak no temporary resources. +- [ ] Previously skipped UI-related tests are either active or replaced with + equivalent coverage. + +--- + +## Milestone 10: Physical-device and cross-client release gate + +**Status:** Not started +**Progress:** 0% + +### Goal + +Validate the behavior that JVM, Robolectric, and emulator tests cannot prove: +real radios, background limits, device interoperability, and compatibility with +released clients. + +### TODO checklist + +#### Device matrix and harness + +- [ ] Define a minimum physical-device matrix covering at least two Android API + levels and two manufacturers. +- [ ] Include devices supporting BLE only and BLE plus Wi-Fi Aware where + available. +- [ ] Build a test control channel that does not interfere with mesh transport. +- [ ] Capture structured traces, packet IDs, connection transitions, and failure + reasons. +- [ ] Make test accounts, identities, and files disposable and non-personal. +- [ ] Provide deterministic scenario setup and cleanup. + +#### Android-to-Android scenarios + +- [ ] Discover, connect, exchange announcements, disconnect, and reconnect. +- [ ] Send public and private messages in both directions. +- [ ] Verify delivery and read receipts. +- [ ] Transfer image, audio, and generic files at multiple sizes. +- [ ] Relay across at least three devices. +- [ ] Partition the mesh and verify store-and-forward delivery after reconnect. +- [ ] Disable and re-enable Bluetooth during active transfers. +- [ ] Lock screens and background both apps during active mesh operation. +- [ ] Kill and recreate one process. +- [ ] Exercise simultaneous connections and duplicate-link resolution. +- [ ] Exercise Wi-Fi Aware failover where supported. + +#### Cross-client and backward compatibility + +- [ ] Test current Android against the current iOS client. +- [ ] Test the rewrite against the last supported Android release. +- [ ] Test legacy announcements without capabilities. +- [ ] Test current capability announcements with an older client. +- [ ] Test canonical private-media type and decode-only prerelease alias. +- [ ] Compare packet, message, identity, fragment, sync, and file golden vectors + across implementations. +- [ ] Verify malformed and unauthenticated inputs are rejected consistently. +- [ ] Verify Nostr fallback messages and receipts across clients. + +#### Background and endurance + +- [ ] Run a multi-hour discovery/connect/disconnect soak test. +- [ ] Run repeated large-transfer and cancellation cycles. +- [ ] Monitor memory, threads, file descriptors, wake locks, and battery impact. +- [ ] Test foreground-service survival with screens off. +- [ ] Test network and Tor availability changes during Nostr operation. +- [ ] Confirm shutdown releases radios, sockets, jobs, and wake locks. + +### Acceptance criteria + +- [ ] All mandatory scenarios pass on the defined device matrix. +- [ ] Android/iOS and old/new clients exchange every supported critical payload. +- [ ] No endurance run shows unbounded growth or leaked resources. +- [ ] Failures produce sufficient traces for deterministic reproduction where + possible. +- [ ] Release approval records the client versions, device matrix, and results. + +--- + +## CI rollout + +### Pull-request gate + +- [ ] Run formatting and static analysis. +- [ ] Run deterministic JVM unit tests. +- [ ] Run bounded property/fuzz tests. +- [ ] Run stable Robolectric tests. +- [ ] Run `clientRewriteContractTest`. +- [ ] Upload JUnit and coverage reports. +- [ ] Reject new failures, errors, or unexpected skips. +- [ ] Reject golden-vector changes without the protocol-change review label. + +### Main and nightly gate + +- [ ] Run the extended fuzz corpus. +- [ ] Run emulator instrumented tests. +- [ ] Run local relay/Tor integration tests. +- [ ] Run physical-device smoke tests when the lab is available. +- [ ] Track runtime, flakes, coverage, and quarantined tests. + +### Release-candidate gate + +- [ ] Run the complete device matrix. +- [ ] Run Android/iOS and old/new interoperability. +- [ ] Run endurance and background scenarios. +- [ ] Review all skipped or quarantined tests. +- [ ] Archive coverage, test, trace, and version metadata with the release. + +## Progress update procedure + +When work lands: + +1. Check completed TODOs in the relevant milestone. +2. Update the milestone percentage based on completed checklist items. +3. Change status to **In progress** when its first TODO is complete. +4. Change status to **Complete** only when all acceptance criteria are met. +5. Update the top-level progress table and milestone completion count. +6. Link the implementing pull request or commit next to material completed work + without including personal information. +7. Record intentionally deferred items and their justification; do not mark them + complete. + +## Final definition of done + +The test program is complete when: + +- [ ] Milestones 0–10 meet every acceptance criterion. +- [ ] Project and package-level line/branch coverage no longer regress. +- [ ] All critical parsers have adversarial and fuzz coverage. +- [ ] All security-sensitive state transitions fail closed under tampering, + replay, downgrade, and corruption. +- [ ] Transport, sync, and lifecycle tests cover disconnection, cancellation, + timeout, and restart. +- [ ] Physical-device and cross-client scenarios pass for every release. +- [ ] The full rewrite can replace the existing implementation while preserving + the unchanged compatibility and acceptance tests. diff --git a/docs/testing-conventions.md b/docs/testing-conventions.md new file mode 100644 index 00000000..b2da4e7b --- /dev/null +++ b/docs/testing-conventions.md @@ -0,0 +1,90 @@ +# Testing conventions + +## Purpose + +These conventions keep the client-rewrite suite deterministic, reproducible, +and portable across implementations. + +## Test locations + +| Test type | Location | Naming | +|---|---|---| +| JVM unit and contract tests | `app/src/test/` | `*Test.kt` | +| Shared deterministic fakes and fixtures | `app/src/test/**/testsupport/` | Descriptive fixture name | +| Robolectric tests | `app/src/test/` | `*RobolectricTest.kt` | +| Android instrumented tests | `app/src/androidTest/` | `*InstrumentedTest.kt` | +| Coverage-tool tests | `tools/coverage/` | `test_*.py` | +| Interoperability fixtures | `app/src/test/resources/contracts/` | Protocol and version in filename | + +## Required behavior + +- Tests must not use arbitrary sleeps. Advance a fake clock or coroutine test + scheduler instead. +- Tests must not require public relays, internet access, Bluetooth hardware, or a + user's persisted data. +- Time, randomness, dispatchers, storage, and transports must be injectable in + code exercised by state-machine tests. +- Randomized failures must print a reproduction seed. Use `TEST_SEED` for a + specific replay. +- Mutable byte arrays returned by fixtures and fakes must be defensively copied. +- Negative security tests must assert fail-closed behavior. +- Protocol round trips must be paired with literal golden vectors for critical + externally visible formats. +- Asynchronous tests must have a deterministic completion condition and a + bounded timeout. +- A fixed bug must retain its smallest reproducing input as a regression test. + +## Naming + +Test names should describe observable behavior: + +```kotlin +@Test +fun `replayed ciphertext is rejected without advancing receive state`() { + // ... +} +``` + +Avoid names tied to private methods or temporary implementation structure. + +## Fixtures and seeds + +Reusable Kotlin fixtures live under +`com.bitchat.android.testsupport`. `ReproducibleTestSeed` resolves +`TEST_SEED` and provides a reproduction hint: + +```sh +TEST_SEED=12345 ./gradlew clientRewriteContractTest +``` + +Never use production keys, contact information, messages, or other user data in +fixtures. + +## Coverage + +Run the full report and non-regression floor: + +```sh +./gradlew clientRewriteContractTest +``` + +Reports are written to: + +- `app/build/reports/jacoco/jacocoTestReport/jacocoTestReport.xml` +- `app/build/reports/jacoco/jacocoTestReport/html/` + +Check executable production lines changed from the base branch: + +```sh +COVERAGE_BASE_REF=origin/main ./gradlew checkChangedLineCoverage +``` + +Generated resource classes, Compose-generated singleton classes, platform +bridges, and vendored Noise code are excluded from first-party coverage metrics. + +## Quarantine and skips + +- A flaky test must be fixed, not silently retried. +- A temporary quarantine must include an issue and removal condition. +- Unexpected skips fail review. Existing skips must be restored or replaced by + equivalent coverage. diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 00000000..b48e1209 --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1 @@ +"""Repository-local verification tooling.""" diff --git a/tools/coverage/__init__.py b/tools/coverage/__init__.py new file mode 100644 index 00000000..8aec01cc --- /dev/null +++ b/tools/coverage/__init__.py @@ -0,0 +1 @@ +"""Coverage verification helpers.""" diff --git a/tools/coverage/check_changed_coverage.py b/tools/coverage/check_changed_coverage.py new file mode 100644 index 00000000..fde7b1bc --- /dev/null +++ b/tools/coverage/check_changed_coverage.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Enforce JaCoCo coverage for executable Kotlin/Java lines changed from a Git base.""" + +from __future__ import annotations + +import argparse +import pathlib +import re +import subprocess +import sys +import xml.etree.ElementTree as ET +from dataclasses import dataclass + + +SOURCE_ROOTS = ( + "app/src/main/java/", + "app/src/main/kotlin/", +) + + +@dataclass(frozen=True) +class CoverageLine: + missed_instructions: int + covered_instructions: int + + @property + def covered(self) -> bool: + return self.covered_instructions > 0 + + +def parse_jacoco(xml_path: pathlib.Path) -> dict[tuple[str, int], CoverageLine]: + root = ET.parse(xml_path).getroot() + result: dict[tuple[str, int], CoverageLine] = {} + for package in root.findall("package"): + package_name = package.attrib["name"] + for source_file in package.findall("sourcefile"): + relative_path = f"{package_name}/{source_file.attrib['name']}" + for line in source_file.findall("line"): + result[(relative_path, int(line.attrib["nr"]))] = CoverageLine( + missed_instructions=int(line.attrib.get("mi", "0")), + covered_instructions=int(line.attrib.get("ci", "0")), + ) + return result + + +def parse_changed_lines(diff_text: str) -> dict[str, set[int]]: + changed: dict[str, set[int]] = {} + current_path: str | None = None + for raw_line in diff_text.splitlines(): + if raw_line.startswith("+++ b/"): + current_path = raw_line[6:] + continue + if raw_line.startswith("+++ /dev/null"): + current_path = None + continue + if not raw_line.startswith("@@") or current_path is None: + continue + match = re.search(r"\+(\d+)(?:,(\d+))?", raw_line) + if match is None: + continue + start = int(match.group(1)) + count = int(match.group(2) or "1") + if count > 0: + changed.setdefault(current_path, set()).update(range(start, start + count)) + return changed + + +def jacoco_relative_path(repository_path: str) -> str | None: + for root in SOURCE_ROOTS: + if repository_path.startswith(root): + return repository_path[len(root) :] + return None + + +def git_diff_command(base: str) -> list[str]: + return [ + "git", + "diff", + # Reformatting an executable line without changing its tokens must not turn an otherwise + # covered change set into a coverage failure. + "--ignore-all-space", + "--unified=0", + "--diff-filter=AM", + base, + "--", + *SOURCE_ROOTS, + ] + + +def git_diff(base: str) -> str: + command = git_diff_command(base) + result = subprocess.run(command, check=True, capture_output=True, text=True) + return result.stdout + + +def evaluate( + changed: dict[str, set[int]], + coverage: dict[tuple[str, int], CoverageLine], +) -> tuple[int, int, list[str]]: + executable = 0 + covered = 0 + missed: list[str] = [] + for repository_path, line_numbers in sorted(changed.items()): + source_path = jacoco_relative_path(repository_path) + if source_path is None: + continue + for line_number in sorted(line_numbers): + line = coverage.get((source_path, line_number)) + if line is None: + continue + executable += 1 + if line.covered: + covered += 1 + else: + missed.append(f"{repository_path}:{line_number}") + return covered, executable, missed + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--xml", type=pathlib.Path, required=True) + parser.add_argument("--base", required=True) + parser.add_argument("--threshold", type=float, default=0.80) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if not 0.0 <= args.threshold <= 1.0: + raise SystemExit("--threshold must be between 0 and 1") + if not args.xml.is_file(): + raise SystemExit(f"JaCoCo XML report not found: {args.xml}") + + coverage = parse_jacoco(args.xml) + changed = parse_changed_lines(git_diff(args.base)) + covered, executable, missed = evaluate(changed, coverage) + ratio = 1.0 if executable == 0 else covered / executable + + print( + f"Changed executable line coverage: {covered}/{executable} " + f"({ratio:.1%}), required {args.threshold:.1%}" + ) + if ratio >= args.threshold: + return 0 + + for location in missed[:50]: + print(f"UNCOVERED {location}") + if len(missed) > 50: + print(f"... and {len(missed) - 50} more uncovered executable lines") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/coverage/test_check_changed_coverage.py b/tools/coverage/test_check_changed_coverage.py new file mode 100644 index 00000000..7c452449 --- /dev/null +++ b/tools/coverage/test_check_changed_coverage.py @@ -0,0 +1,85 @@ +import pathlib +import tempfile +import unittest + +from tools.coverage.check_changed_coverage import ( + CoverageLine, + evaluate, + git_diff_command, + jacoco_relative_path, + parse_changed_lines, + parse_jacoco, +) + + +class ChangedCoverageToolTest(unittest.TestCase): + def test_parses_added_and_modified_hunks(self) -> None: + diff = """\ +diff --git a/app/src/main/java/example/Thing.kt b/app/src/main/java/example/Thing.kt +--- a/app/src/main/java/example/Thing.kt ++++ b/app/src/main/java/example/Thing.kt +@@ -1,0 +2,3 @@ ++a ++b ++c +@@ -9 +12 @@ +-old ++new +""" + self.assertEqual( + {"app/src/main/java/example/Thing.kt": {2, 3, 4, 12}}, + parse_changed_lines(diff), + ) + + def test_parses_jacoco_source_lines(self) -> None: + xml = """\ + + + + + + + + +""" + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "report.xml" + path.write_text(xml, encoding="utf-8") + parsed = parse_jacoco(path) + + self.assertTrue(parsed[("example/Thing.kt", 2)].covered) + self.assertFalse(parsed[("example/Thing.kt", 3)].covered) + + def test_evaluates_only_executable_changed_lines(self) -> None: + changed = {"app/src/main/java/example/Thing.kt": {1, 2, 3}} + coverage = { + ("example/Thing.kt", 2): CoverageLine(0, 1), + ("example/Thing.kt", 3): CoverageLine(1, 0), + } + + self.assertEqual( + (1, 2, ["app/src/main/java/example/Thing.kt:3"]), + evaluate(changed, coverage), + ) + + def test_maps_both_supported_source_roots(self) -> None: + self.assertEqual( + "example/Thing.kt", + jacoco_relative_path("app/src/main/java/example/Thing.kt"), + ) + self.assertEqual( + "example/Thing.kt", + jacoco_relative_path("app/src/main/kotlin/example/Thing.kt"), + ) + self.assertIsNone(jacoco_relative_path("app/src/test/example/Thing.kt")) + + def test_changed_line_diff_ignores_formatting_only_edits(self) -> None: + command = git_diff_command("origin/main") + + self.assertIn("--ignore-all-space", command) + self.assertIn("--unified=0", command) + self.assertEqual("origin/main", command[command.index("--diff-filter=AM") + 1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/release_gate/__init__.py b/tools/release_gate/__init__.py new file mode 100644 index 00000000..e9ab0855 --- /dev/null +++ b/tools/release_gate/__init__.py @@ -0,0 +1 @@ +"""Physical-device and cross-client release-gate tooling.""" diff --git a/tools/release_gate/android_lab.py b/tools/release_gate/android_lab.py new file mode 100644 index 00000000..165bb469 --- /dev/null +++ b/tools/release_gate/android_lab.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""USB/ADB control helpers for the physical release gate. + +Device selectors are accepted only as ephemeral command inputs. They are never +printed or written to artifacts; all output uses the operator-assigned alias. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Callable + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +if str(REPOSITORY_ROOT) not in sys.path: + sys.path.insert(0, str(REPOSITORY_ROOT)) + +from tools.release_gate.release_gate import ( + GateError, + SAFE_ID_RE, + append_trace_event, +) + + +APPLICATION_ID = "com.bitchat.droid" + + +def find_adb() -> str: + direct = shutil.which("adb") + if direct: + return direct + android_home = os.environ.get("ANDROID_HOME") + if android_home: + candidate = Path(android_home) / "platform-tools" / "adb" + if candidate.is_file(): + return str(candidate) + raise GateError("adb was not found; set ANDROID_HOME or add adb to PATH") + + +def run_adb( + serial: str, + arguments: list[str], + *, + runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, +) -> str: + result = runner( + [find_adb(), "-s", serial, *arguments], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + if result.returncode != 0: + raise GateError("ADB command failed for the selected logical device") + return result.stdout.strip() + + +def count_connected_devices( + *, + runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, +) -> int: + result = runner( + [find_adb(), "devices"], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + if result.returncode != 0: + raise GateError("could not enumerate ADB devices") + return sum( + 1 + for line in result.stdout.splitlines()[1:] + if line.strip().endswith("\tdevice") + ) + + +def probe_device(serial: str, alias: str) -> dict[str, object]: + if not SAFE_ID_RE.fullmatch(alias): + raise GateError("device alias must be a lowercase logical identifier") + api_text = run_adb(serial, ["shell", "getprop", "ro.build.version.sdk"]) + if not api_text.isdigit(): + raise GateError("selected device returned an invalid API level") + features = run_adb(serial, ["shell", "pm", "list", "features"]) + manufacturer = run_adb( + serial, ["shell", "getprop", "ro.product.manufacturer"] + ).strip().lower() + model = run_adb(serial, ["shell", "getprop", "ro.product.model"]).strip() + capabilities = ["ble-central", "ble-peripheral"] + if "android.hardware.wifi.aware" in features: + capabilities.append("wifi-aware") + return { + "alias": alias, + "platform": "android", + "model": model, + "manufacturer_class": re.sub(r"[^a-z0-9._-]", "-", manufacturer)[:64], + "api_level": int(api_text), + "physical": True, + "capabilities": capabilities, + } + + +def prepare_disposable_device(serial: str, confirmed: bool) -> None: + if not confirmed: + raise GateError("prepare requires --confirm-disposable-app-data") + run_adb(serial, ["shell", "am", "force-stop", APPLICATION_ID]) + output = run_adb(serial, ["shell", "pm", "clear", APPLICATION_ID]) + if "Success" not in output: + raise GateError("could not clear disposable app data") + + +def collect_resource_snapshot(serial: str) -> dict[str, int | bool]: + pid_text = run_adb(serial, ["shell", "pidof", APPLICATION_ID]) + pid = pid_text.split()[0] if pid_text else "" + metrics: dict[str, int | bool] = {"process-running": bool(pid)} + if not pid.isdigit(): + return metrics + meminfo = run_adb(serial, ["shell", "dumpsys", "meminfo", APPLICATION_ID]) + total_match = re.search(r"TOTAL\s+(\d+)", meminfo) + metrics["total-pss-kb"] = int(total_match.group(1)) if total_match else -1 + thread_text = run_adb( + serial, + ["shell", "sh", "-c", f"find /proc/{pid}/task -mindepth 1 -maxdepth 1 | wc -l"], + ) + fd_text = run_adb( + serial, + ["shell", "sh", "-c", f"find /proc/{pid}/fd -mindepth 1 -maxdepth 1 | wc -l"], + ) + metrics["thread-count"] = int(thread_text) if thread_text.isdigit() else -1 + metrics["fd-count"] = int(fd_text) if fd_text.isdigit() else -1 + power = run_adb(serial, ["shell", "dumpsys", "power"]) + metrics["app-wakelock-count"] = sum( + 1 + for line in power.splitlines() + if APPLICATION_ID in line and "WakeLock" in line + ) + battery = run_adb(serial, ["shell", "dumpsys", "battery"]) + battery_level = re.search(r"^\s*level:\s*(\d+)", battery, re.MULTILINE) + metrics["battery-level-percent"] = ( + int(battery_level.group(1)) if battery_level else -1 + ) + return metrics + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + + commands.add_parser("count") + + probe = commands.add_parser("probe") + probe.add_argument("--serial", required=True, help=argparse.SUPPRESS) + probe.add_argument("--alias", required=True) + + prepare = commands.add_parser("prepare") + prepare.add_argument("--serial", required=True, help=argparse.SUPPRESS) + prepare.add_argument("--confirm-disposable-app-data", action="store_true") + + cleanup = commands.add_parser("cleanup") + cleanup.add_argument("--serial", required=True, help=argparse.SUPPRESS) + cleanup.add_argument("--confirm-disposable-app-data", action="store_true") + + snapshot = commands.add_parser("snapshot") + snapshot.add_argument("--serial", required=True, help=argparse.SUPPRESS) + snapshot.add_argument("--alias", required=True) + snapshot.add_argument("--run", type=Path, required=True) + snapshot.add_argument("--scenario", required=True) + snapshot.add_argument("--event", default="resource-snapshot") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + if args.command == "count": + print(json.dumps({"authorized-device-count": count_connected_devices()})) + elif args.command == "probe": + print(json.dumps(probe_device(args.serial, args.alias), sort_keys=True)) + elif args.command in {"prepare", "cleanup"}: + prepare_disposable_device( + args.serial, args.confirm_disposable_app_data + ) + print(json.dumps({"status": "clean", "application": APPLICATION_ID})) + elif args.command == "snapshot": + metrics = collect_resource_snapshot(args.serial) + append_trace_event( + args.run, + args.scenario, + args.alias, + args.event, + "observed", + None, + metrics, + ) + print(json.dumps({"source_alias": args.alias, "metrics": metrics}, sort_keys=True)) + return 0 + except (GateError, OSError, subprocess.SubprocessError) as error: + print(f"android lab error: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/release_gate/device-matrix.example.json b/tools/release_gate/device-matrix.example.json new file mode 100644 index 00000000..01b8f629 --- /dev/null +++ b/tools/release_gate/device-matrix.example.json @@ -0,0 +1,72 @@ +{ + "schema_version": 1, + "commit": "0000000000000000000000000000000000000000", + "clients": { + "android-current": { + "version": "replace-with-release-candidate", + "commit": "0000000000000000000000000000000000000000" + }, + "android-legacy": { + "version": "replace-with-last-supported-release" + }, + "ios-current": { + "version": "replace-with-current-ios-release" + } + }, + "lab_capabilities": ["local-relay", "tor"], + "devices": [ + { + "alias": "android-low", + "platform": "android", + "model": "replace-with-model", + "manufacturer_class": "manufacturer-a", + "api_level": 28, + "physical": true, + "capabilities": ["ble-central", "ble-peripheral"] + }, + { + "alias": "android-current", + "platform": "android", + "model": "replace-with-model", + "manufacturer_class": "manufacturer-b", + "api_level": 35, + "physical": true, + "capabilities": ["ble-central", "ble-peripheral", "wifi-aware"] + }, + { + "alias": "android-relay", + "platform": "android", + "model": "replace-with-model", + "manufacturer_class": "manufacturer-c", + "api_level": 33, + "physical": true, + "capabilities": ["ble-central", "ble-peripheral"] + }, + { + "alias": "android-aware", + "platform": "android", + "model": "replace-with-model", + "manufacturer_class": "manufacturer-b", + "api_level": 35, + "physical": true, + "capabilities": ["ble-central", "ble-peripheral", "wifi-aware"] + }, + { + "alias": "android-legacy", + "platform": "android", + "model": "replace-with-model", + "manufacturer_class": "manufacturer-a", + "api_level": 28, + "physical": true, + "capabilities": ["ble-central", "ble-peripheral"] + }, + { + "alias": "ios-current", + "platform": "ios", + "model": "replace-with-model", + "manufacturer_class": "apple", + "physical": true, + "capabilities": ["ble-central", "ble-peripheral"] + } + ] +} diff --git a/tools/release_gate/release_gate.py b/tools/release_gate/release_gate.py new file mode 100644 index 00000000..fbec7e32 --- /dev/null +++ b/tools/release_gate/release_gate.py @@ -0,0 +1,744 @@ +#!/usr/bin/env python3 +"""Create, record, validate, and archive the physical release gate. + +The host-side CLI is the control channel. It coordinates operators over USB or +local files and never sends control traffic through the mesh under test. +Artifacts intentionally contain logical device aliases and aggregate evidence, +not device identifiers, addresses, peer IDs, or message contents. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import re +import sys +import zipfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + + +SCHEMA_VERSION = 1 +RESULT_STATUSES = {"pending", "pass", "fail", "blocked", "unsupported"} +COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") +SAFE_ID_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$") +DISALLOWED_KEYS = { + "serial", + "serial_number", + "udid", + "imei", + "bluetooth_address", + "mac_address", + "ip_address", + "peer_id", + "username", + "user_name", + "email", + "account", + "device_name", +} +HASH_VALUE_KEYS = { + "commit", + "sha256", + "digest", + "scenario_manifest_sha256", + "fixture_sha256", + "vector_manifest_digest", + "corpus_digest", +} +SENSITIVE_PATTERNS = ( + re.compile(r"(?:^|[\s/])Users/[^/\s]+"), + re.compile(r"(?:^|[\s/])home/[^/\s]+"), + re.compile(r"\b[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}\b"), + re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"), + re.compile(r"\b(?:[0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}\b"), + re.compile(r"\b[0-9A-Fa-f]{16,}\b"), +) +MAX_FILE_SIZE_BYTES = 50 * 1024 * 1024 + + +class GateError(ValueError): + """A release-gate artifact violated an executable contract.""" + + +def utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def load_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise GateError(f"could not read JSON {path.name}: {error}") from error + if not isinstance(value, dict): + raise GateError(f"{path.name} must contain a JSON object") + return value + + +def write_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + temporary.replace(path) + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_json_digest(value: dict[str, Any]) -> str: + encoded = json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8") + return sha256_bytes(encoded) + + +def validate_privacy(value: Any, path: tuple[str, ...] = ()) -> None: + if isinstance(value, dict): + for key, child in value.items(): + normalized = str(key).lower() + if normalized in DISALLOWED_KEYS: + raise GateError(f"disallowed identifying field: {'.'.join(path + (str(key),))}") + validate_privacy(child, path + (str(key),)) + return + if isinstance(value, list): + for index, child in enumerate(value): + validate_privacy(child, path + (str(index),)) + return + if not isinstance(value, str): + return + final_key = path[-1].lower().replace("-", "_") if path else "" + for index, pattern in enumerate(SENSITIVE_PATTERNS): + if index == len(SENSITIVE_PATTERNS) - 1 and ( + final_key in HASH_VALUE_KEYS + or final_key.endswith("_digest") + or final_key.endswith("_sha256") + ): + continue + if pattern.search(value): + raise GateError(f"potential identifying value at {'.'.join(path)}") + + +def validate_manifest(manifest: dict[str, Any]) -> dict[str, dict[str, Any]]: + if manifest.get("schema_version") != SCHEMA_VERSION: + raise GateError("unsupported scenario schema_version") + scenarios = manifest.get("scenarios") + if not isinstance(scenarios, list) or not scenarios: + raise GateError("scenario manifest must contain scenarios") + by_id: dict[str, dict[str, Any]] = {} + for scenario in scenarios: + if not isinstance(scenario, dict): + raise GateError("each scenario must be an object") + scenario_id = scenario.get("id") + if not isinstance(scenario_id, str) or not re.fullmatch( + r"[A-Z0-9]{3}-\d{3}", scenario_id + ): + raise GateError(f"invalid scenario id: {scenario_id!r}") + if scenario_id in by_id: + raise GateError(f"duplicate scenario id: {scenario_id}") + if scenario.get("category") not in { + "transport", + "android-platform", + "android-to-android", + "cross-client", + "endurance", + }: + raise GateError(f"invalid category for {scenario_id}") + if scenario.get("required") is not True: + raise GateError(f"release scenario {scenario_id} must be required") + if not scenario.get("participants") or not scenario.get("evidence"): + raise GateError(f"scenario {scenario_id} lacks participants or evidence") + by_id[scenario_id] = scenario + validate_privacy(manifest) + return by_id + + +def validate_matrix( + matrix: dict[str, Any], + expected_commit: str | None = None, + *, + allow_placeholders: bool = False, +) -> dict[str, dict[str, Any]]: + if matrix.get("schema_version") != SCHEMA_VERSION: + raise GateError("unsupported matrix schema_version") + commit = matrix.get("commit") + if not isinstance(commit, str) or not COMMIT_RE.fullmatch(commit): + raise GateError("matrix commit must be a lowercase full Git commit") + if expected_commit is not None and commit != expected_commit: + raise GateError("matrix commit does not match the release candidate") + clients = matrix.get("clients") + required_clients = {"android-current", "android-legacy", "ios-current"} + if not isinstance(clients, dict) or not required_clients.issubset(clients): + raise GateError("matrix must version current Android, legacy Android, and current iOS") + for client_name in required_clients: + client = clients.get(client_name) + if not isinstance(client, dict) or not isinstance(client.get("version"), str): + raise GateError(f"{client_name} must declare a version") + if not allow_placeholders and client["version"].startswith("replace-with-"): + raise GateError(f"{client_name} still contains a template version") + current_commit = clients["android-current"].get("commit") + if current_commit != commit and not (allow_placeholders and commit == "0" * 40): + raise GateError("current Android client commit must match the matrix commit") + lab_capabilities = matrix.get("lab_capabilities") + if not isinstance(lab_capabilities, list) or any( + not isinstance(capability, str) or not SAFE_ID_RE.fullmatch(capability) + for capability in lab_capabilities + ): + raise GateError("matrix must declare logical lab_capabilities") + devices = matrix.get("devices") + if not isinstance(devices, list): + raise GateError("matrix devices must be a list") + by_alias: dict[str, dict[str, Any]] = {} + for device in devices: + if not isinstance(device, dict): + raise GateError("each device must be an object") + alias = device.get("alias") + if not isinstance(alias, str) or not SAFE_ID_RE.fullmatch(alias): + raise GateError(f"invalid logical device alias: {alias!r}") + if alias in by_alias: + raise GateError(f"duplicate device alias: {alias}") + if device.get("physical") is not True: + raise GateError(f"{alias} is not a physical device") + if device.get("platform") not in {"android", "ios"}: + raise GateError(f"{alias} has an unsupported platform") + if not isinstance(device.get("capabilities"), list): + raise GateError(f"{alias} must declare capabilities") + if not allow_placeholders and ( + not isinstance(device.get("model"), str) + or device["model"].startswith("replace-with-") + ): + raise GateError(f"{alias} still contains template values") + by_alias[alias] = device + android = [device for device in devices if device.get("platform") == "android"] + if len(android) < 3: + raise GateError("three physical Android devices are required for relay scenarios") + api_levels = {device.get("api_level") for device in android} + manufacturers = {device.get("manufacturer_class") for device in android} + if len(api_levels) < 2 or not all(isinstance(level, int) for level in api_levels): + raise GateError("Android matrix must cover at least two API levels") + if len(manufacturers) < 2 or None in manufacturers: + raise GateError("Android matrix must cover at least two manufacturer classes") + for device in android: + capabilities = set(device["capabilities"]) + if not {"ble-central", "ble-peripheral"}.issubset(capabilities): + raise GateError(f"{device['alias']} lacks required BLE roles") + if not any("wifi-aware" in device["capabilities"] for device in android): + raise GateError("at least one Android device must support Wi-Fi Aware") + if not any(device.get("platform") == "ios" for device in devices): + raise GateError("a physical iOS device is required") + validate_privacy(matrix) + return by_alias + + +def _fixture_bytes(seed: bytes, size: int) -> bytes: + output = bytearray() + counter = 0 + while len(output) < size: + output.extend(hashlib.sha256(seed + counter.to_bytes(4, "big")).digest()) + counter += 1 + return bytes(output[:size]) + + +def create_fixtures(directory: Path, run_id: str) -> dict[str, Any]: + fixture_directory = directory / "fixtures" + fixture_directory.mkdir() + definitions = ( + ("empty.bin", 0, False), + ("small.bin", 4 * 1024, False), + ("lab-résumé-秘密.bin", 256 * 1024, False), + ("maximum.bin", MAX_FILE_SIZE_BYTES, True), + ("oversized.bin", MAX_FILE_SIZE_BYTES + 1, True), + ) + fixtures: list[dict[str, Any]] = [] + for name, size, sparse in definitions: + path = fixture_directory / name + if sparse: + with path.open("wb") as stream: + stream.truncate(size) + else: + path.write_bytes(_fixture_bytes(run_id.encode("utf-8"), size)) + fixtures.append( + { + "name": name, + "size_bytes": size, + "sparse": sparse, + "fixture_sha256": sha256_file(path), + } + ) + manifest = {"schema_version": SCHEMA_VERSION, "fixtures": fixtures} + write_json(fixture_directory / "manifest.json", manifest) + return manifest + + +def initialize_run( + manifest: dict[str, Any], + matrix: dict[str, Any], + output: Path, + commit: str, + run_id: str, + *, + started_at: str | None = None, +) -> dict[str, Any]: + scenarios = validate_manifest(manifest) + devices = validate_matrix(matrix, commit) + if not SAFE_ID_RE.fullmatch(run_id): + raise GateError("run id must be a non-identifying lowercase logical id") + missing_aliases = { + participant + for scenario in scenarios.values() + for participant in scenario["participants"] + if participant not in devices + } + if missing_aliases: + raise GateError(f"matrix lacks scenario aliases: {sorted(missing_aliases)}") + for scenario_id, scenario in scenarios.items(): + available = set(matrix["lab_capabilities"]) + for participant in scenario["participants"]: + available.update(devices[participant]["capabilities"]) + missing_capabilities = set(scenario["capabilities"]) - available + if missing_capabilities: + raise GateError( + f"{scenario_id} lacks capabilities: {sorted(missing_capabilities)}" + ) + output.mkdir(parents=True, exist_ok=False) + write_json(output / "manifest.json", manifest) + write_json(output / "matrix.json", matrix) + fixtures = create_fixtures(output, run_id) + results = { + "schema_version": SCHEMA_VERSION, + "run_id": run_id, + "commit": commit, + "started_at": started_at or utc_now(), + "completed_at": None, + "scenario_manifest_sha256": canonical_json_digest(manifest), + "fixture_manifest_sha256": canonical_json_digest(fixtures), + "scenario_results": { + scenario_id: { + "status": "pending", + "participants": scenario["participants"], + "evidence": {}, + "reason_code": None, + "updated_at": None, + "history": [], + } + for scenario_id, scenario in scenarios.items() + }, + } + write_json(output / "results.json", results) + (output / "trace.jsonl").write_text("", encoding="utf-8") + return results + + +def _parse_scalar(value: str) -> Any: + lowered = value.lower() + if lowered in {"true", "false"}: + return lowered == "true" + try: + return int(value) + except ValueError: + try: + return float(value) + except ValueError: + return value + + +def parse_evidence(values: Iterable[str]) -> dict[str, Any]: + evidence: dict[str, Any] = {} + for value in values: + if "=" not in value: + raise GateError("evidence must use key=value") + key, raw = value.split("=", 1) + if not SAFE_ID_RE.fullmatch(key): + raise GateError(f"invalid evidence key: {key!r}") + evidence[key] = _parse_scalar(raw) + validate_privacy(evidence) + return evidence + + +def record_result( + run_directory: Path, + scenario_id: str, + status: str, + evidence: dict[str, Any], + reason_code: str | None, + *, + updated_at: str | None = None, +) -> dict[str, Any]: + manifest = load_json(run_directory / "manifest.json") + scenarios = validate_manifest(manifest) + if scenario_id not in scenarios: + raise GateError(f"unknown scenario: {scenario_id}") + if status not in RESULT_STATUSES - {"pending"}: + raise GateError(f"invalid terminal status: {status}") + if reason_code is not None and not SAFE_ID_RE.fullmatch(reason_code): + raise GateError("reason code must be a non-identifying stable code") + if status == "pass" and not evidence: + raise GateError("passing a scenario requires structured evidence") + if status == "pass" and reason_code is not None: + raise GateError("passing a scenario cannot have a failure reason code") + if status in {"fail", "blocked", "unsupported"} and reason_code is None: + raise GateError(f"{status} requires a stable reason code") + if any(not SAFE_ID_RE.fullmatch(str(key)) for key in evidence): + raise GateError("evidence keys must be stable logical identifiers") + validate_privacy(evidence) + results = load_json(run_directory / "results.json") + result = results["scenario_results"][scenario_id] + terminal_update = { + "status": status, + "evidence": evidence, + "reason_code": reason_code, + "updated_at": updated_at or utc_now(), + } + result.setdefault("history", []).append(terminal_update.copy()) + result.update( + terminal_update + ) + results["completed_at"] = ( + result["updated_at"] + if all( + item.get("status") == "pass" + for item in results["scenario_results"].values() + ) + else None + ) + write_json(run_directory / "results.json", results) + return results + + +def append_trace_event( + run_directory: Path, + scenario_id: str, + source_alias: str, + event: str, + outcome: str, + reason_code: str | None, + metrics: dict[str, Any], + *, + timestamp: str | None = None, +) -> dict[str, Any]: + manifest = load_json(run_directory / "manifest.json") + matrix = load_json(run_directory / "matrix.json") + scenarios = validate_manifest(manifest) + devices = validate_matrix(matrix, allow_placeholders=False) + if scenario_id not in scenarios: + raise GateError(f"unknown scenario: {scenario_id}") + if source_alias not in devices: + raise GateError(f"unknown source alias: {source_alias}") + for label, value in (("event", event), ("outcome", outcome)): + if not SAFE_ID_RE.fullmatch(value): + raise GateError(f"invalid {label}") + if reason_code is not None and not SAFE_ID_RE.fullmatch(reason_code): + raise GateError("invalid reason code") + if any(not SAFE_ID_RE.fullmatch(str(key)) for key in metrics): + raise GateError("trace metric keys must be stable logical identifiers") + if any( + not isinstance(value, (int, float, bool)) + or isinstance(value, float) and not math.isfinite(value) + for value in metrics.values() + ): + raise GateError("trace metrics must be numeric or boolean aggregates") + trace = { + "timestamp": timestamp or utc_now(), + "scenario_id": scenario_id, + "source_alias": source_alias, + "event": event, + "outcome": outcome, + "reason_code": reason_code, + "metrics": metrics, + } + validate_privacy(trace) + with (run_directory / "trace.jsonl").open("a", encoding="utf-8") as stream: + stream.write(json.dumps(trace, sort_keys=True, ensure_ascii=False) + "\n") + return trace + + +def _validate_trace( + run_directory: Path, + scenarios: dict[str, dict[str, Any]], + devices: dict[str, dict[str, Any]], +) -> tuple[int, set[str]]: + path = run_directory / "trace.jsonl" + if not path.exists(): + raise GateError("trace.jsonl is missing") + count = 0 + traced_scenarios: set[str] = set() + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + try: + event = json.loads(line) + except json.JSONDecodeError as error: + raise GateError(f"invalid trace line {line_number}") from error + if not isinstance(event, dict): + raise GateError(f"trace line {line_number} must be an object") + if event.get("scenario_id") not in scenarios: + raise GateError(f"trace line {line_number} has an unknown scenario") + if event.get("source_alias") not in devices: + raise GateError(f"trace line {line_number} has an unknown source") + if set(event) != { + "timestamp", + "scenario_id", + "source_alias", + "event", + "outcome", + "reason_code", + "metrics", + }: + raise GateError(f"trace line {line_number} has unexpected fields") + if not isinstance(event.get("metrics"), dict) or any( + not isinstance(value, (int, float, bool)) + for value in event["metrics"].values() + ): + raise GateError(f"trace line {line_number} has invalid metrics") + validate_privacy(event) + count += 1 + traced_scenarios.add(event["scenario_id"]) + return count, traced_scenarios + + +def validate_run(run_directory: Path, *, allow_incomplete: bool = False) -> dict[str, Any]: + manifest = load_json(run_directory / "manifest.json") + matrix = load_json(run_directory / "matrix.json") + results = load_json(run_directory / "results.json") + scenarios = validate_manifest(manifest) + devices = validate_matrix(matrix, results.get("commit")) + if results.get("schema_version") != SCHEMA_VERSION: + raise GateError("unsupported results schema_version") + if results.get("scenario_manifest_sha256") != canonical_json_digest(manifest): + raise GateError("scenario manifest changed after run initialization") + fixture_manifest = load_json(run_directory / "fixtures" / "manifest.json") + if results.get("fixture_manifest_sha256") != canonical_json_digest(fixture_manifest): + raise GateError("fixture manifest changed after run initialization") + actual = results.get("scenario_results") + if not isinstance(actual, dict) or set(actual) != set(scenarios): + raise GateError("results must contain exactly every declared scenario") + validate_privacy(results) + for scenario_id, scenario in scenarios.items(): + result = actual[scenario_id] + if result.get("participants") != scenario["participants"]: + raise GateError(f"{scenario_id} participants changed") + if any(alias not in devices for alias in result["participants"]): + raise GateError(f"{scenario_id} references an unknown device") + available = set(matrix["lab_capabilities"]) + for participant in result["participants"]: + available.update(devices[participant]["capabilities"]) + missing_capabilities = set(scenario["capabilities"]) - available + if missing_capabilities: + raise GateError( + f"{scenario_id} lacks capabilities: {sorted(missing_capabilities)}" + ) + status = result.get("status") + if status not in RESULT_STATUSES: + raise GateError(f"{scenario_id} has invalid status") + if not allow_incomplete and status != "pass": + raise GateError(f"{scenario_id} is not passing: {status}") + if status == "pass": + evidence = result.get("evidence") + missing = set(scenario["evidence"]) - set(evidence or {}) + if missing: + raise GateError(f"{scenario_id} lacks evidence: {sorted(missing)}") + if scenario.get("minimum_duration_minutes") is not None and ( + evidence.get("duration-minutes", 0) + < scenario["minimum_duration_minutes"] + ): + raise GateError(f"{scenario_id} did not meet minimum duration") + if scenario.get("minimum_cycles") is not None and ( + evidence.get("cycle-count", 0) < scenario["minimum_cycles"] + ): + raise GateError(f"{scenario_id} did not meet minimum cycles") + trace_events, traced_scenarios = _validate_trace(run_directory, scenarios, devices) + if not allow_incomplete: + missing_traces = set(scenarios) - traced_scenarios + if missing_traces: + raise GateError( + f"passing scenarios lack structured traces: {sorted(missing_traces)}" + ) + if not results.get("completed_at"): + raise GateError("complete results must record completed_at") + summary = { + status: sum( + 1 for result in actual.values() if result.get("status") == status + ) + for status in sorted(RESULT_STATUSES) + } + summary["trace_events"] = trace_events + summary["complete"] = all( + result.get("status") == "pass" for result in actual.values() + ) + return summary + + +def render_summary(run_directory: Path) -> str: + results = load_json(run_directory / "results.json") + summary = validate_run(run_directory, allow_incomplete=True) + rows = [ + "# Physical release-gate result", + "", + f"- Run: `{results['run_id']}`", + f"- Commit: `{results['commit']}`", + f"- Complete: `{str(summary['complete']).lower()}`", + f"- Trace events: {summary['trace_events']}", + "", + "| Status | Count |", + "|---|---:|", + ] + rows.extend( + f"| {status} | {summary[status]} |" for status in sorted(RESULT_STATUSES) + ) + return "\n".join(rows) + "\n" + + +def create_bundle(run_directory: Path, output: Path) -> None: + validate_run(run_directory) + if output.exists(): + raise GateError("refusing to overwrite an existing release-gate bundle") + members = [ + Path("manifest.json"), + Path("matrix.json"), + Path("results.json"), + Path("trace.jsonl"), + Path("fixtures/manifest.json"), + ] + generated = {"summary.md": render_summary(run_directory).encode("utf-8")} + checksums: list[str] = [] + payloads: dict[str, bytes] = {} + for member in members: + payload = (run_directory / member).read_bytes() + payloads[member.as_posix()] = payload + payloads.update(generated) + for name in sorted(payloads): + checksums.append(f"{sha256_bytes(payloads[name])} {name}") + payloads["SHA256SUMS"] = ("\n".join(checksums) + "\n").encode("utf-8") + output.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for name in sorted(payloads): + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o100644 << 16 + archive.writestr(info, payloads[name]) + + +def _default_manifest() -> Path: + return Path(__file__).with_name("scenarios.json") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + manifest = subparsers.add_parser("validate-manifest") + manifest.add_argument("--manifest", type=Path, default=_default_manifest()) + + matrix = subparsers.add_parser("validate-matrix") + matrix.add_argument("--matrix", type=Path, required=True) + matrix.add_argument("--commit") + matrix.add_argument("--allow-template", action="store_true") + + initialize = subparsers.add_parser("init") + initialize.add_argument("--manifest", type=Path, default=_default_manifest()) + initialize.add_argument("--matrix", type=Path, required=True) + initialize.add_argument("--output", type=Path, required=True) + initialize.add_argument("--commit", required=True) + initialize.add_argument("--run-id", required=True) + + record = subparsers.add_parser("record") + record.add_argument("--run", type=Path, required=True) + record.add_argument("--scenario", required=True) + record.add_argument("--status", choices=sorted(RESULT_STATUSES - {"pending"}), required=True) + record.add_argument("--evidence", action="append", default=[]) + record.add_argument("--reason-code") + + trace = subparsers.add_parser("trace") + trace.add_argument("--run", type=Path, required=True) + trace.add_argument("--scenario", required=True) + trace.add_argument("--source", required=True) + trace.add_argument("--event", required=True) + trace.add_argument("--outcome", required=True) + trace.add_argument("--reason-code") + trace.add_argument("--metric", action="append", default=[]) + + validate = subparsers.add_parser("validate") + validate.add_argument("--run", type=Path, required=True) + validate.add_argument("--allow-incomplete", action="store_true") + + summary = subparsers.add_parser("summary") + summary.add_argument("--run", type=Path, required=True) + + bundle = subparsers.add_parser("bundle") + bundle.add_argument("--run", type=Path, required=True) + bundle.add_argument("--output", type=Path, required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + if args.command == "validate-manifest": + scenarios = validate_manifest(load_json(args.manifest)) + print(f"valid scenarios: {len(scenarios)}") + elif args.command == "validate-matrix": + devices = validate_matrix( + load_json(args.matrix), + args.commit, + allow_placeholders=args.allow_template, + ) + print(f"valid devices: {len(devices)}") + elif args.command == "init": + initialize_run( + load_json(args.manifest), + load_json(args.matrix), + args.output, + args.commit, + args.run_id, + ) + print(args.output) + elif args.command == "record": + record_result( + args.run, + args.scenario, + args.status, + parse_evidence(args.evidence), + args.reason_code, + ) + elif args.command == "trace": + append_trace_event( + args.run, + args.scenario, + args.source, + args.event, + args.outcome, + args.reason_code, + parse_evidence(args.metric), + ) + elif args.command == "validate": + print( + json.dumps( + validate_run(args.run, allow_incomplete=args.allow_incomplete), + sort_keys=True, + ) + ) + elif args.command == "summary": + print(render_summary(args.run), end="") + elif args.command == "bundle": + create_bundle(args.run, args.output) + print(args.output) + return 0 + except GateError as error: + print(f"release gate error: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/release_gate/scenarios.json b/tools/release_gate/scenarios.json new file mode 100644 index 00000000..d9b50907 --- /dev/null +++ b/tools/release_gate/scenarios.json @@ -0,0 +1,251 @@ +{ + "schema_version": 1, + "scenario_version": "1.0", + "scenarios": [ + { + "id": "TRN-001", + "category": "transport", + "title": "Complete the physical BLE, GATT, MTU, Wi-Fi Aware, and failover matrix", + "participants": ["android-low", "android-current", "android-aware"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral", "wifi-aware"], + "evidence": ["matrix-check-count", "mtu-case-count", "failure-injection-count", "shutdown-check-count"] + }, + { + "id": "AND-001", + "category": "android-platform", + "title": "Complete API-level, manufacturer, permission-revocation, and background gates", + "participants": ["android-low", "android-current", "android-aware"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral", "wifi-aware"], + "evidence": ["api-level-count", "manufacturer-count", "permission-revocation-count", "background-case-count"] + }, + { + "id": "A2A-001", + "category": "android-to-android", + "title": "Discover, connect, announce, disconnect, and reconnect", + "participants": ["android-low", "android-current"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["connection-transitions", "packet-correlation-count", "failure-reasons"] + }, + { + "id": "A2A-002", + "category": "android-to-android", + "title": "Exchange public and private messages in both directions", + "participants": ["android-low", "android-current"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["message-counts", "packet-correlation-count"] + }, + { + "id": "A2A-003", + "category": "android-to-android", + "title": "Advance delivery and read receipts", + "participants": ["android-low", "android-current"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["receipt-transitions"] + }, + { + "id": "A2A-004", + "category": "android-to-android", + "title": "Transfer image, audio, and generic files at multiple sizes", + "participants": ["android-low", "android-current"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["fixture-digests", "transfer-progress", "delivery-digests"] + }, + { + "id": "A2A-005", + "category": "android-to-android", + "title": "Relay across three Android devices", + "participants": ["android-low", "android-current", "android-relay"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["route-transitions", "ttl-values", "packet-correlation-count"] + }, + { + "id": "A2A-006", + "category": "android-to-android", + "title": "Partition and recover store-and-forward delivery", + "participants": ["android-low", "android-current", "android-relay"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["partition-window", "queue-counts", "delivery-counts"] + }, + { + "id": "A2A-007", + "category": "android-to-android", + "title": "Toggle Bluetooth during active transfers", + "participants": ["android-low", "android-current"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["radio-transitions", "transfer-terminal-state"] + }, + { + "id": "A2A-008", + "category": "android-to-android", + "title": "Lock screens and background both apps during mesh operation", + "participants": ["android-low", "android-current"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["lifecycle-transitions", "service-state", "delivery-counts"] + }, + { + "id": "A2A-009", + "category": "android-to-android", + "title": "Kill and recreate one process", + "participants": ["android-low", "android-current"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["process-generation", "restored-state", "reconnect-count"] + }, + { + "id": "A2A-010", + "category": "android-to-android", + "title": "Resolve simultaneous and duplicate links", + "participants": ["android-low", "android-current"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["candidate-links", "canonical-link-count"] + }, + { + "id": "A2A-011", + "category": "android-to-android", + "title": "Fail over through Wi-Fi Aware", + "participants": ["android-current", "android-aware"], + "required": true, + "capabilities": ["wifi-aware"], + "evidence": ["transport-selection", "failover-window", "delivery-counts"] + }, + { + "id": "XCL-001", + "category": "cross-client", + "title": "Current Android interoperates with current iOS", + "participants": ["android-current", "ios-current"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["client-versions", "payload-counts"] + }, + { + "id": "XCL-002", + "category": "cross-client", + "title": "Current rewrite contracts interoperate with the last supported Android", + "participants": ["android-current", "android-legacy"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["client-versions", "payload-counts"] + }, + { + "id": "XCL-003", + "category": "cross-client", + "title": "Legacy announcements without capabilities remain compatible", + "participants": ["android-current", "android-legacy"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["announcement-version", "peer-state"] + }, + { + "id": "XCL-004", + "category": "cross-client", + "title": "Older clients safely ignore current capability announcements", + "participants": ["android-current", "android-legacy"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["announcement-version", "peer-state"] + }, + { + "id": "XCL-005", + "category": "cross-client", + "title": "Canonical private media and prerelease decode-only alias interoperate", + "participants": ["android-current", "ios-current"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["wire-type", "delivery-digests", "downgrade-decision"] + }, + { + "id": "XCL-006", + "category": "cross-client", + "title": "Golden vectors match across implementations", + "participants": ["android-current", "ios-current", "android-legacy"], + "required": true, + "capabilities": [], + "evidence": ["vector-manifest-digest", "comparison-counts"] + }, + { + "id": "XCL-007", + "category": "cross-client", + "title": "Malformed and unauthenticated inputs are rejected consistently", + "participants": ["android-current", "ios-current", "android-legacy"], + "required": true, + "capabilities": [], + "evidence": ["corpus-digest", "rejection-counts", "crash-count"] + }, + { + "id": "XCL-008", + "category": "cross-client", + "title": "Nostr fallback messages and receipts interoperate", + "participants": ["android-current", "ios-current"], + "required": true, + "capabilities": ["local-relay", "tor"], + "evidence": ["relay-fixture", "event-id-count", "receipt-counts"] + }, + { + "id": "END-001", + "category": "endurance", + "title": "Multi-hour discovery, connect, and disconnect soak", + "participants": ["android-low", "android-current"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "minimum_duration_minutes": 240, + "evidence": ["duration-minutes", "connection-counts", "failure-counts"] + }, + { + "id": "END-002", + "category": "endurance", + "title": "Repeated large-transfer and cancellation cycles", + "participants": ["android-low", "android-current"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "minimum_cycles": 50, + "evidence": ["cycle-count", "delivery-counts", "cancellation-counts"] + }, + { + "id": "END-003", + "category": "endurance", + "title": "Resource growth remains bounded", + "participants": ["android-low", "android-current"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["memory-samples", "thread-samples", "fd-samples", "wakelock-samples", "battery-samples"] + }, + { + "id": "END-004", + "category": "endurance", + "title": "Foreground service survives with screens off", + "participants": ["android-low", "android-current"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["service-state", "screen-state", "delivery-counts"] + }, + { + "id": "END-005", + "category": "endurance", + "title": "Nostr survives network and Tor availability changes", + "participants": ["android-current"], + "required": true, + "capabilities": ["local-relay", "tor"], + "evidence": ["network-transitions", "tor-transitions", "receipt-counts"] + }, + { + "id": "END-006", + "category": "endurance", + "title": "Shutdown releases radios, sockets, jobs, and wake locks", + "participants": ["android-low", "android-current"], + "required": true, + "capabilities": ["ble-central", "ble-peripheral"], + "evidence": ["resource-terminal-state", "late-callback-count"] + } + ] +} diff --git a/tools/release_gate/test_release_gate.py b/tools/release_gate/test_release_gate.py new file mode 100644 index 00000000..3ca39ac8 --- /dev/null +++ b/tools/release_gate/test_release_gate.py @@ -0,0 +1,367 @@ +import json +import subprocess +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest import mock + +from tools.release_gate import android_lab +from tools.release_gate.release_gate import ( + GateError, + append_trace_event, + canonical_json_digest, + create_bundle, + initialize_run, + load_json, + parse_evidence, + record_result, + validate_manifest, + validate_matrix, + validate_privacy, + validate_run, +) + + +COMMIT = "1" * 40 +TOOL_DIRECTORY = Path(__file__).parent + + +def valid_matrix(): + return { + "schema_version": 1, + "commit": COMMIT, + "clients": { + "android-current": {"version": "2.0.0-rc1", "commit": COMMIT}, + "android-legacy": {"version": "1.9.0"}, + "ios-current": {"version": "2.0.0"}, + }, + "lab_capabilities": ["local-relay", "tor"], + "devices": [ + { + "alias": "android-low", + "platform": "android", + "model": "model-low", + "manufacturer_class": "vendor-a", + "api_level": 28, + "physical": True, + "capabilities": ["ble-central", "ble-peripheral"], + }, + { + "alias": "android-current", + "platform": "android", + "model": "model-current", + "manufacturer_class": "vendor-b", + "api_level": 35, + "physical": True, + "capabilities": ["ble-central", "ble-peripheral", "wifi-aware"], + }, + { + "alias": "android-relay", + "platform": "android", + "model": "model-relay", + "manufacturer_class": "vendor-c", + "api_level": 33, + "physical": True, + "capabilities": ["ble-central", "ble-peripheral"], + }, + { + "alias": "android-aware", + "platform": "android", + "model": "model-aware", + "manufacturer_class": "vendor-b", + "api_level": 35, + "physical": True, + "capabilities": ["ble-central", "ble-peripheral", "wifi-aware"], + }, + { + "alias": "android-legacy", + "platform": "android", + "model": "model-legacy", + "manufacturer_class": "vendor-a", + "api_level": 28, + "physical": True, + "capabilities": ["ble-central", "ble-peripheral"], + }, + { + "alias": "ios-current", + "platform": "ios", + "model": "ios-model", + "manufacturer_class": "apple", + "physical": True, + "capabilities": ["ble-central", "ble-peripheral"], + }, + ], + } + + +class ReleaseGateTest(unittest.TestCase): + def setUp(self): + self.manifest = load_json(TOOL_DIRECTORY / "scenarios.json") + + def test_manifest_covers_every_release_category(self): + scenarios = validate_manifest(self.manifest) + self.assertEqual(27, len(scenarios)) + self.assertEqual( + { + "transport", + "android-platform", + "android-to-android", + "cross-client", + "endurance", + }, + {scenario["category"] for scenario in scenarios.values()}, + ) + + def test_documented_matrix_template_is_schema_valid_but_not_runnable(self): + template = load_json(TOOL_DIRECTORY / "device-matrix.example.json") + self.assertEqual(6, len(validate_matrix(template, allow_placeholders=True))) + with self.assertRaises(GateError): + validate_matrix(template) + + def test_matrix_enforces_distinct_api_manufacturer_and_counterpart_clients(self): + matrix = valid_matrix() + self.assertEqual(6, len(validate_matrix(matrix, COMMIT))) + + for device in matrix["devices"]: + if device["platform"] == "android": + device["manufacturer_class"] = "one-vendor" + with self.assertRaisesRegex(GateError, "manufacturer"): + validate_matrix(matrix, COMMIT) + + def test_privacy_policy_rejects_identifiers_paths_addresses_and_long_ids(self): + rejected = ( + {"serial": "device-selector"}, + {"note": "/" + "home/operator/result"}, + {"note": "operator@example.test"}, + {"note": "192.0.2.1"}, + {"note": "aa:bb:cc:dd:ee:ff"}, + {"note": "0123456789abcdef"}, + ) + for value in rejected: + with self.subTest(value=value), self.assertRaises(GateError): + validate_privacy(value) + validate_privacy({"commit": COMMIT, "packet-correlation-count": 3}) + + def test_initialize_creates_disposable_fixtures_and_pending_results(self): + with tempfile.TemporaryDirectory() as temporary: + run = Path(temporary) / "rc-run" + results = initialize_run( + self.manifest, + valid_matrix(), + run, + COMMIT, + "rc-run", + started_at="2026-01-01T00:00:00+00:00", + ) + self.assertTrue(all( + result["status"] == "pending" + for result in results["scenario_results"].values() + )) + fixtures = load_json(run / "fixtures" / "manifest.json")["fixtures"] + sizes = {fixture["name"]: fixture["size_bytes"] for fixture in fixtures} + self.assertEqual(0, sizes["empty.bin"]) + self.assertEqual(50 * 1024 * 1024, sizes["maximum.bin"]) + self.assertEqual(50 * 1024 * 1024 + 1, sizes["oversized.bin"]) + self.assertEqual( + results["scenario_manifest_sha256"], + canonical_json_digest(self.manifest), + ) + summary = validate_run(run, allow_incomplete=True) + self.assertEqual(27, summary["pending"]) + self.assertFalse(summary["complete"]) + + def test_record_requires_structured_evidence_and_rejects_pii(self): + with tempfile.TemporaryDirectory() as temporary: + run = Path(temporary) / "rc-run" + initialize_run(self.manifest, valid_matrix(), run, COMMIT, "rc-run") + with self.assertRaises(GateError): + record_result(run, "A2A-001", "pass", {}, None) + with self.assertRaises(GateError): + record_result( + run, + "A2A-001", + "pass", + {"connection-transitions": "operator@example.test"}, + None, + ) + with self.assertRaisesRegex(GateError, "reason code"): + record_result(run, "A2A-001", "fail", {}, None) + recorded = record_result( + run, + "A2A-001", + "blocked", + {}, + "counterpart-unavailable", + updated_at="2026-01-01T00:00:00+00:00", + ) + self.assertEqual( + ["blocked"], + [ + item["status"] + for item in recorded["scenario_results"]["A2A-001"]["history"] + ], + ) + + def test_complete_run_requires_all_evidence_traces_and_endurance_bounds(self): + with tempfile.TemporaryDirectory() as temporary: + run = Path(temporary) / "rc-run" + initialize_run(self.manifest, valid_matrix(), run, COMMIT, "rc-run") + scenarios = validate_manifest(self.manifest) + for scenario_id, scenario in scenarios.items(): + evidence = {key: 1 for key in scenario["evidence"]} + if "duration-minutes" in evidence: + evidence["duration-minutes"] = 240 + if "cycle-count" in evidence: + evidence["cycle-count"] = 50 + record_result( + run, + scenario_id, + "pass", + evidence, + None, + updated_at="2026-01-01T04:00:00+00:00", + ) + append_trace_event( + run, + scenario_id, + scenario["participants"][0], + "scenario-terminal", + "pass", + None, + {"assertion-count": len(evidence)}, + timestamp="2026-01-01T04:00:00+00:00", + ) + summary = validate_run(run) + self.assertTrue(summary["complete"]) + self.assertEqual(27, summary["pass"]) + self.assertEqual(27, summary["trace_events"]) + + bundle = Path(temporary) / "release-gate.zip" + create_bundle(run, bundle) + with zipfile.ZipFile(bundle) as archive: + self.assertEqual( + { + "SHA256SUMS", + "fixtures/manifest.json", + "manifest.json", + "matrix.json", + "results.json", + "summary.md", + "trace.jsonl", + }, + set(archive.namelist()), + ) + with self.assertRaisesRegex(GateError, "overwrite"): + create_bundle(run, bundle) + + def test_manifest_tampering_after_initialization_is_detected(self): + with tempfile.TemporaryDirectory() as temporary: + run = Path(temporary) / "rc-run" + initialize_run(self.manifest, valid_matrix(), run, COMMIT, "rc-run") + manifest = load_json(run / "manifest.json") + manifest["scenario_version"] = "tampered" + (run / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + with self.assertRaisesRegex(GateError, "changed"): + validate_run(run, allow_incomplete=True) + + def test_trace_accepts_only_aggregate_metrics(self): + with tempfile.TemporaryDirectory() as temporary: + run = Path(temporary) / "rc-run" + initialize_run(self.manifest, valid_matrix(), run, COMMIT, "rc-run") + with self.assertRaisesRegex(GateError, "numeric"): + append_trace_event( + run, + "A2A-001", + "android-current", + "packet", + "observed", + None, + {"raw-packet": "payload"}, + ) + + def test_evidence_parser_is_typed_and_privacy_checked(self): + self.assertEqual( + {"count": 3, "ratio": 0.5, "clean": True}, + parse_evidence(["count=3", "ratio=0.5", "clean=true"]), + ) + with self.assertRaises(GateError): + parse_evidence(["note=10.0.0.1"]) + + @mock.patch("tools.release_gate.android_lab.find_adb", return_value="adb") + def test_adb_device_count_never_returns_selectors(self, _find_adb): + completed = subprocess.CompletedProcess( + ["adb", "devices"], + 0, + "List of devices attached\nselector-one\tdevice\nselector-two\toffline\n", + "", + ) + count = android_lab.count_connected_devices(runner=lambda *args, **kwargs: completed) + self.assertEqual(1, count) + + @mock.patch("tools.release_gate.android_lab.run_adb") + def test_adb_probe_emits_only_logical_device_metadata(self, run_adb): + run_adb.side_effect = [ + "35", + "feature:android.hardware.wifi.aware", + "Vendor", + "Model", + ] + probe = android_lab.probe_device("ephemeral-selector", "android-current") + self.assertEqual("android-current", probe["alias"]) + self.assertNotIn("serial", probe) + self.assertIn("wifi-aware", probe["capabilities"]) + + @mock.patch("tools.release_gate.android_lab.run_adb") + def test_disposable_cleanup_targets_the_real_application_id(self, run_adb): + run_adb.side_effect = ["", "Success"] + + android_lab.prepare_disposable_device("ephemeral-selector", confirmed=True) + + self.assertEqual( + [ + mock.call( + "ephemeral-selector", + ["shell", "am", "force-stop", "com.bitchat.droid"], + ), + mock.call( + "ephemeral-selector", + ["shell", "pm", "clear", "com.bitchat.droid"], + ), + ], + run_adb.call_args_list, + ) + + @mock.patch("tools.release_gate.android_lab.run_adb") + def test_resource_snapshot_returns_only_aggregate_metrics(self, run_adb): + run_adb.side_effect = [ + "123", + "TOTAL 2048", + "7", + "11", + "WakeLock com.bitchat.droid\nWakeLock another.package", + "level: 73", + ] + metrics = android_lab.collect_resource_snapshot("ephemeral-selector") + self.assertEqual( + { + "process-running": True, + "total-pss-kb": 2048, + "thread-count": 7, + "fd-count": 11, + "app-wakelock-count": 1, + "battery-level-percent": 73, + }, + metrics, + ) + self.assertEqual( + mock.call( + "ephemeral-selector", + ["shell", "pidof", "com.bitchat.droid"], + ), + run_adb.call_args_list[0], + ) + + +if __name__ == "__main__": + unittest.main()