apunkt/nyx
1
0
Fork 0
mirror of https://github.com/elicpeter/nyx.git synced 2026-06-09 19:45:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

refactor(dynamic): extend Kafka protocol emulator with binary protocol support, Pubsub gRPC emulator, and enhance listener and endpoint handling

Browse source
This commit is contained in:
elipeter 2026-05-27 14:11:31 -05:00
parent 030b054843
commit 1a0e2d204b
6 changed files with 1530 additions and 106 deletions
Download patch file Download diff file Expand all files Collapse all files

53
Cargo.lock generated
View file

@ -637,6 +637,12 @@ dependencies = [
"num-traits",
]
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
version = "0.1.5"
@ -741,6 +747,25 @@ dependencies = [
"regex-syntax",
]
[[package]]
name = "h2"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733"
dependencies = [
"atomic-waker",
"bytes",
"fnv",
"futures-core",
"futures-sink",
"http",
"indexmap",
"slab",
"tokio",
"tokio-util",
"tracing",
]
[[package]]
name = "half"
version = "2.7.1"
@ -1142,6 +1167,7 @@ dependencies = [
"axum",
"bitflags",
"blake3",
"bytes",
"bytesize",
"chrono",
"clap",
@ -1151,6 +1177,8 @@ dependencies = [
"dashmap",
"directories",
"glob",
"h2",
"http",
"ignore",
"indicatif",
"num_cpus",
@ -1159,6 +1187,7 @@ dependencies = [
"petgraph",
"phf",
"predicates",
"prost",
"r2d2",
"r2d2_sqlite",
"rayon",
@ -1413,6 +1442,29 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "prost"
version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568"
dependencies = [
"bytes",
"prost-derive",
]
[[package]]
name = "prost-derive"
version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
dependencies = [
"anyhow",
"itertools",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "quote"
version = "1.0.45"
@ -1925,6 +1977,7 @@ version = "1.52.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
dependencies = [
"bytes",
"libc",
"mio",
"pin-project-lite",

8
Cargo.toml
View file

@ -48,7 +48,7 @@ smt-system-z3 = ["dep:z3"]
docgen = []
# Dynamic verification layer: builds harnesses from findings, runs them in a
# sandbox, reports back whether the sink fires.
dynamic = ["dep:tempfile"]
dynamic = ["dep:bytes", "dep:h2", "dep:http", "dep:prost", "dep:tempfile", "dep:tokio"]
# Phase 19 (Track E.3): the `nyx-image-builder` helper binary that builds
# and pins per-toolchain Docker images. Gated so it does not bloat the
# default `nyx` build with extra TOML-write logic CI-only operators need.
@ -141,7 +141,11 @@ smallvec = { version = "1.15.1", features = ["serde"] }
rustc-hash = "2.1.2"
uuid = { version = "1.23.1", features = ["v4"] }
axum = { version = "0.8.9", optional = true }
tokio = { version = "1.52.3", features = ["rt-multi-thread", "macros", "signal", "sync"], optional = true }
bytes = { version = "1.11.0", optional = true }
h2 = { version = "0.4.14", optional = true }
http = { version = "1.3.1", optional = true }
prost = { version = "0.14.3", optional = true }
tokio = { version = "1.52.3", features = ["rt-multi-thread", "macros", "signal", "sync", "net", "io-util"], optional = true }
tokio-stream = { version = "0.1.18", features = ["sync"], optional = true }
tower-http = { version = "0.6.10", features = ["cors", "compression-gzip", "trace", "set-header", "limit"], optional = true }
z3 = { version = "0.20.0", optional = true}

16
src/dynamic/lang/java.rs
View file

@ -4192,7 +4192,6 @@ public class NyxHarness {{
java.util.Properties consumerProps = new java.util.Properties();
consumerProps.put("bootstrap.servers", bootstrap);
consumerProps.put("group.id", "nyx-" + Long.toString(System.nanoTime()));
consumerProps.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
consumerProps.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
consumerProps.put("auto.offset.reset", "earliest");
@ -4209,8 +4208,18 @@ public class NyxHarness {{
.invoke(future, Long.valueOf(2L), java.util.concurrent.TimeUnit.SECONDS);
producerClass.getMethod("flush").invoke(producer);
consumerClass.getMethod("subscribe", java.util.Collection.class)
.invoke(consumer, java.util.Collections.singletonList(topic));
Class<?> topicPartitionClass = Class.forName("org.apache.kafka.common.TopicPartition");
Object partition = topicPartitionClass.getConstructor(String.class, int.class)
.newInstance(topic, Integer.valueOf(0));
java.util.List<Object> partitions = java.util.Collections.singletonList(partition);
consumerClass.getMethod("assign", java.util.Collection.class).invoke(consumer, partitions);
try {{
consumerClass.getMethod("seekToBeginning", java.util.Collection.class)
.invoke(consumer, partitions);
}} catch (Throwable seekError) {{
consumerClass.getMethod("seek", topicPartitionClass, long.class)
.invoke(consumer, partition, Long.valueOf(0L));
}}
Object records = consumerClass.getMethod("poll", java.time.Duration.class)
.invoke(consumer, java.time.Duration.ofSeconds(2));
if (!(records instanceof Iterable)) {{
@ -4233,7 +4242,6 @@ public class NyxHarness {{
System.err.println("NYX_EXCEPTION: " + c.getClass().getName() + ": " + c.getMessage());
}}
if (success) {{
consumerClass.getMethod("commitSync").invoke(consumer);
nyxRecordBrokerEvent("NYX_KAFKA_LOG", "ack", topic, Long.toString(offset));
}}
delivered = true;

22
src/dynamic/lang/python.rs
View file

@ -1085,7 +1085,7 @@ def _nyx_try_real_kafka(topic, body, handler_name):
if not bootstrap:
return False
try:
from kafka import KafkaConsumer, KafkaProducer
from kafka import KafkaConsumer, KafkaProducer, TopicPartition
except Exception:
return False
_h = getattr(_entry_mod, handler_name, None)
@ -1104,9 +1104,8 @@ def _nyx_try_real_kafka(topic, body, handler_name):
retries=0,
)
_consumer = KafkaConsumer(
str(topic),
bootstrap_servers=[bootstrap],
group_id="nyx-" + str(os.getpid()),
group_id=None,
auto_offset_reset="earliest",
enable_auto_commit=False,
consumer_timeout_ms=2000,
@ -1118,6 +1117,23 @@ def _nyx_try_real_kafka(topic, body, handler_name):
_nyx_record_broker_publish("NYX_KAFKA_LOG", topic, body)
_producer.send(str(topic), body).get(timeout=2)
_producer.flush(timeout=2)
_tp = TopicPartition(str(topic), 0)
_consumer.assign([_tp])
try:
_consumer.seek_to_beginning(_tp)
except Exception:
_consumer.seek(_tp, 0)
_records = _consumer.poll(timeout_ms=2000, max_records=1)
for _partition_records in _records.values():
for _record in _partition_records:
_nyx_record_broker_event("NYX_KAFKA_LOG", "deliver", topic, _record.value)
_h(_record.value)
try:
_consumer.commit()
except Exception:
pass
_nyx_record_broker_event("NYX_KAFKA_LOG", "ack", topic, str(getattr(_record, "offset", "")))
return True
for _record in _consumer:
_nyx_record_broker_event("NYX_KAFKA_LOG", "deliver", topic, _record.value)
_h(_record.value)

1535
src/dynamic/stubs/broker.rs
View file

File diff suppressed because it is too large Load diff

2
tests/message_handler_corpus.rs
View file

@ -577,7 +577,7 @@ fn message_handler_remaining_brokers_emit_delivery_and_ack_events() {
}
#[test]
fn message_handler_remaining_brokers_try_http_emulators_before_loopback() {
fn message_handler_remaining_brokers_keep_http_fallbacks_after_real_clients() {
let cases = [
(
Lang::Python,