**refactor(dynamic): add Kafka HTTP emulator with publish/poll/commit support, extend endpoint rewriting and stub event recording across Java, Python, and Rust**

This commit is contained in:
elipeter 2026-05-27 11:01:46 -05:00
parent 433036aead
commit 57d3677bd4
7 changed files with 564 additions and 40 deletions

View file

@ -3873,26 +3873,29 @@ fn emit_message_handler_harness(
JavaBroker::Kafka => (
crate::dynamic::stubs::KAFKA_PUBLISH_MARKER,
format!(
r#" NyxKafkaLoopback brokerRef = new NyxKafkaLoopback();
System.out.println({publish_marker:?} + " " + {queue:?});
nyxRecordBrokerPublish("NYX_KAFKA_LOG", {queue:?}, payload);
brokerRef.publish({queue:?}, payload);
for (NyxKafkaRecord rec : brokerRef.poll({queue:?}, 1)) {{
nyxRecordBrokerEvent("NYX_KAFKA_LOG", "deliver", {queue:?}, rec.value);
System.out.println("__NYX_SINK_HIT__");
boolean success = false;
try {{
java.lang.reflect.Method m = entryInst.getClass().getDeclaredMethod({handler:?}, String.class);
m.setAccessible(true);
m.invoke(entryInst, rec.value);
success = true;
}} catch (Exception e) {{
Throwable c = (e instanceof java.lang.reflect.InvocationTargetException && e.getCause() != null) ? e.getCause() : e;
System.err.println("NYX_EXCEPTION: " + c.getClass().getName() + ": " + c.getMessage());
}}
if (success) {{
brokerRef.commit(rec);
nyxRecordBrokerEvent("NYX_KAFKA_LOG", "ack", {queue:?}, Long.toString(rec.offset));
r#" if (!nyxTryRealKafkaClient({queue:?}, payload, entryInst, {handler:?})
&& !nyxTryKafkaHttp({queue:?}, payload, entryInst, {handler:?})) {{
NyxKafkaLoopback brokerRef = new NyxKafkaLoopback();
System.out.println({publish_marker:?} + " " + {queue:?});
nyxRecordBrokerPublish("NYX_KAFKA_LOG", {queue:?}, payload);
brokerRef.publish({queue:?}, payload);
for (NyxKafkaRecord rec : brokerRef.poll({queue:?}, 1)) {{
nyxRecordBrokerEvent("NYX_KAFKA_LOG", "deliver", {queue:?}, rec.value);
System.out.println("__NYX_SINK_HIT__");
boolean success = false;
try {{
java.lang.reflect.Method m = entryInst.getClass().getDeclaredMethod({handler:?}, String.class);
m.setAccessible(true);
m.invoke(entryInst, rec.value);
success = true;
}} catch (Exception e) {{
Throwable c = (e instanceof java.lang.reflect.InvocationTargetException && e.getCause() != null) ? e.getCause() : e;
System.err.println("NYX_EXCEPTION: " + c.getClass().getName() + ": " + c.getMessage());
}}
if (success) {{
brokerRef.commit(rec);
nyxRecordBrokerEvent("NYX_KAFKA_LOG", "ack", {queue:?}, Long.toString(rec.offset));
}}
}}
}}"#,
handler = handler,
@ -3928,6 +3931,207 @@ public class NyxHarness {{
}}
}}
static boolean nyxTryKafkaHttp(String topic, String payload, Object entryInst, String handler) {{
String endpoint = System.getenv("NYX_KAFKA_ENDPOINT");
if (endpoint == null || !(endpoint.startsWith("http://") || endpoint.startsWith("https://"))) {{
return false;
}}
try {{
String base = endpoint.replaceAll("/+$", "");
String topicPath = java.net.URLEncoder.encode(topic, java.nio.charset.StandardCharsets.UTF_8);
System.out.println({kafka_publish_marker:?} + " " + topic);
nyxHttpRequest(
"POST",
base + "/topics/" + topicPath + "/messages",
payload.getBytes(java.nio.charset.StandardCharsets.UTF_8)
);
String recordsJson = nyxHttpRequest(
"GET",
base + "/topics/" + topicPath + "/records?max=1",
new byte[0]
);
if (recordsJson == null || !recordsJson.contains("\"records\"") || !recordsJson.contains("\"value\"")) {{
return false;
}}
String value = nyxJsonStringField(recordsJson, "value");
String offset = nyxJsonNumberField(recordsJson, "offset");
if (offset == null || offset.isEmpty()) {{
offset = "0";
}}
System.out.println("__NYX_SINK_HIT__");
boolean success = false;
try {{
java.lang.reflect.Method m = entryInst.getClass().getDeclaredMethod(handler, String.class);
m.setAccessible(true);
m.invoke(entryInst, value);
success = true;
}} catch (Exception e) {{
Throwable c = (e instanceof java.lang.reflect.InvocationTargetException && e.getCause() != null) ? e.getCause() : e;
System.err.println("NYX_EXCEPTION: " + c.getClass().getName() + ": " + c.getMessage());
}}
if (success) {{
String body = "offset=" + java.net.URLEncoder.encode(offset, java.nio.charset.StandardCharsets.UTF_8);
nyxHttpRequest(
"POST",
base + "/topics/" + topicPath + "/commit",
body.getBytes(java.nio.charset.StandardCharsets.UTF_8)
);
}}
return true;
}} catch (Throwable e) {{
System.err.println("NYX_KAFKA_HTTP_FALLBACK: " + e.getClass().getName() + ": " + e.getMessage());
return false;
}}
}}
static boolean nyxTryRealKafkaClient(String topic, String payload, Object entryInst, String handler) {{
Object consumer = null;
try {{
Class<?> mockConsumerClass = Class.forName("org.apache.kafka.clients.consumer.MockConsumer");
Class<?> resetClass = Class.forName("org.apache.kafka.clients.consumer.OffsetResetStrategy");
Object earliest = java.lang.Enum.valueOf(resetClass.asSubclass(java.lang.Enum.class), "EARLIEST");
consumer = mockConsumerClass.getConstructor(resetClass).newInstance(earliest);
Class<?> topicPartitionClass = Class.forName("org.apache.kafka.common.TopicPartition");
Object partition = topicPartitionClass.getConstructor(String.class, int.class).newInstance(topic, 0);
java.util.List<Object> partitions = java.util.Collections.singletonList(partition);
mockConsumerClass.getMethod("subscribe", java.util.Collection.class)
.invoke(consumer, java.util.Collections.singletonList(topic));
mockConsumerClass.getMethod("rebalance", java.util.Collection.class).invoke(consumer, partitions);
java.util.Map<Object, Long> beginnings = new java.util.HashMap<>();
beginnings.put(partition, Long.valueOf(0L));
mockConsumerClass.getMethod("updateBeginningOffsets", java.util.Map.class).invoke(consumer, beginnings);
Class<?> recordClass = Class.forName("org.apache.kafka.clients.consumer.ConsumerRecord");
Object record = null;
for (java.lang.reflect.Constructor<?> ctor : recordClass.getConstructors()) {{
if (ctor.getParameterCount() == 5) {{
record = ctor.newInstance(topic, Integer.valueOf(0), Long.valueOf(0L), null, payload);
break;
}}
}}
if (record == null) {{
return false;
}}
System.out.println({kafka_publish_marker:?} + " " + topic);
nyxRecordBrokerPublish("NYX_KAFKA_LOG", topic, payload);
mockConsumerClass.getMethod("addRecord", recordClass).invoke(consumer, record);
Object records = mockConsumerClass.getMethod("poll", java.time.Duration.class)
.invoke(consumer, java.time.Duration.ofMillis(10));
if (!(records instanceof Iterable)) {{
return false;
}}
boolean delivered = false;
for (Object rec : (Iterable<?>) records) {{
String value = String.valueOf(rec.getClass().getMethod("value").invoke(rec));
long offset = ((Number) rec.getClass().getMethod("offset").invoke(rec)).longValue();
nyxRecordBrokerEvent("NYX_KAFKA_LOG", "deliver", topic, value);
System.out.println("__NYX_SINK_HIT__");
boolean success = false;
try {{
java.lang.reflect.Method m = entryInst.getClass().getDeclaredMethod(handler, String.class);
m.setAccessible(true);
m.invoke(entryInst, value);
success = true;
}} catch (Exception e) {{
Throwable c = (e instanceof java.lang.reflect.InvocationTargetException && e.getCause() != null) ? e.getCause() : e;
System.err.println("NYX_EXCEPTION: " + c.getClass().getName() + ": " + c.getMessage());
}}
if (success) {{
Class<?> offsetClass = Class.forName("org.apache.kafka.clients.consumer.OffsetAndMetadata");
Object metadata = offsetClass.getConstructor(long.class).newInstance(Long.valueOf(offset + 1L));
java.util.Map<Object, Object> commits = new java.util.HashMap<>();
commits.put(partition, metadata);
mockConsumerClass.getMethod("commitSync", java.util.Map.class).invoke(consumer, commits);
nyxRecordBrokerEvent("NYX_KAFKA_LOG", "ack", topic, Long.toString(offset));
}}
delivered = true;
}}
return delivered;
}} catch (ClassNotFoundException missingKafkaClient) {{
return false;
}} catch (Throwable e) {{
System.err.println("NYX_REAL_KAFKA_FALLBACK: " + e.getClass().getName() + ": " + e.getMessage());
return false;
}} finally {{
if (consumer instanceof AutoCloseable) {{
try {{
((AutoCloseable) consumer).close();
}} catch (Exception ignored) {{
}}
}}
}}
}}
static String nyxHttpRequest(String method, String target, byte[] body) throws Exception {{
java.net.HttpURLConnection conn = (java.net.HttpURLConnection) java.net.URI.create(target).toURL().openConnection();
conn.setRequestMethod(method);
conn.setConnectTimeout(1000);
conn.setReadTimeout(2000);
if (body != null && body.length > 0) {{
conn.setDoOutput(true);
conn.setRequestProperty("Content-Type", "application/octet-stream");
conn.setRequestProperty("Content-Length", Integer.toString(body.length));
try (java.io.OutputStream os = conn.getOutputStream()) {{
os.write(body);
}}
}}
java.io.InputStream is = conn.getResponseCode() >= 400 ? conn.getErrorStream() : conn.getInputStream();
if (is == null) {{
return "";
}}
try (java.io.InputStream input = is) {{
byte[] data = input.readAllBytes();
return new String(data, java.nio.charset.StandardCharsets.UTF_8);
}} finally {{
conn.disconnect();
}}
}}
static String nyxJsonStringField(String json, String field) {{
String needle = "\"" + field + "\":\"";
int start = json.indexOf(needle);
if (start < 0) return "";
start += needle.length();
StringBuilder out = new StringBuilder();
boolean escaped = false;
for (int i = start; i < json.length(); i++) {{
char ch = json.charAt(i);
if (escaped) {{
switch (ch) {{
case 'n': out.append('\n'); break;
case 'r': out.append('\r'); break;
case 't': out.append('\t'); break;
case '"': out.append('"'); break;
case '\\': out.append('\\'); break;
default: out.append(ch); break;
}}
escaped = false;
}} else if (ch == '\\') {{
escaped = true;
}} else if (ch == '"') {{
break;
}} else {{
out.append(ch);
}}
}}
return out.toString();
}}
static String nyxJsonNumberField(String json, String field) {{
String needle = "\"" + field + "\":";
int start = json.indexOf(needle);
if (start < 0) return "";
start += needle.length();
int end = start;
while (end < json.length() && Character.isDigit(json.charAt(end))) {{
end++;
}}
return json.substring(start, end);
}}
static boolean nyxTryRealSqs(String queue, String payload, Object entryInst, String handler) {{
String endpoint = System.getenv("NYX_SQS_ENDPOINT");
if (endpoint == null || !(endpoint.startsWith("http://") || endpoint.startsWith("https://"))) {{
@ -4052,6 +4256,7 @@ public class NyxHarness {{
"#,
entry_class = entry_class,
dispatch_block = dispatch_block,
kafka_publish_marker = crate::dynamic::stubs::KAFKA_PUBLISH_MARKER,
sqs_publish_marker = crate::dynamic::stubs::SQS_PUBLISH_MARKER,
);
HarnessSource {

View file

@ -1018,22 +1018,23 @@ _chan.basic_publish(exchange="", routing_key={queue:?}, body=payload)"#,
publish_marker = crate::dynamic::stubs::RABBIT_PUBLISH_MARKER,
),
PythonBroker::Kafka => format!(
r#"_loop = NyxKafkaLoopback()
def _nyx_kafka_dispatch(message):
_h = getattr(_entry_mod, {handler:?}, None)
if _h is None:
print("NYX_HANDLER_NOT_FOUND: " + {handler:?}, file=sys.stderr, flush=True)
sys.exit(78)
_h(message)
_loop.subscribe({queue:?}, _nyx_kafka_dispatch)
print({publish_marker:?} + " " + {queue:?}, flush=True)
_nyx_record_broker_publish("NYX_KAFKA_LOG", {queue:?}, payload)
_loop.publish({queue:?}, payload)
for _record in _loop.poll({queue:?}, max_records=1):
_nyx_record_broker_event("NYX_KAFKA_LOG", "deliver", {queue:?}, _record.value)
_nyx_kafka_dispatch(_record.value)
_loop.commit(_record)
_nyx_record_broker_event("NYX_KAFKA_LOG", "ack", {queue:?}, str(_record.offset))"#,
r#"if not _nyx_try_kafka_http({queue:?}, payload, {handler:?}):
_loop = NyxKafkaLoopback()
def _nyx_kafka_dispatch(message):
_h = getattr(_entry_mod, {handler:?}, None)
if _h is None:
print("NYX_HANDLER_NOT_FOUND: " + {handler:?}, file=sys.stderr, flush=True)
sys.exit(78)
_h(message)
_loop.subscribe({queue:?}, _nyx_kafka_dispatch)
print({publish_marker:?} + " " + {queue:?}, flush=True)
_nyx_record_broker_publish("NYX_KAFKA_LOG", {queue:?}, payload)
_loop.publish({queue:?}, payload)
for _record in _loop.poll({queue:?}, max_records=1):
_nyx_record_broker_event("NYX_KAFKA_LOG", "deliver", {queue:?}, _record.value)
_nyx_kafka_dispatch(_record.value)
_loop.commit(_record)
_nyx_record_broker_event("NYX_KAFKA_LOG", "ack", {queue:?}, str(_record.offset))"#,
handler = handler,
queue = queue,
publish_marker = crate::dynamic::stubs::KAFKA_PUBLISH_MARKER,
@ -1064,6 +1065,50 @@ def _nyx_record_broker_event(env_name, action, destination, body):
def _nyx_record_broker_publish(env_name, destination, body):
_nyx_record_broker_event(env_name, "publish", destination, body)
def _nyx_try_kafka_http(topic, body, handler_name):
endpoint = os.environ.get("NYX_KAFKA_ENDPOINT", "")
if not (endpoint.startswith("http://") or endpoint.startswith("https://")):
return False
_h = getattr(_entry_mod, handler_name, None)
if _h is None:
print("NYX_HANDLER_NOT_FOUND: " + handler_name, file=sys.stderr, flush=True)
sys.exit(78)
try:
import json
import urllib.parse
import urllib.request
base = endpoint.rstrip("/")
topic_path = urllib.parse.quote(str(topic), safe="")
print({kafka_publish_marker:?} + " " + str(topic), flush=True)
_send = urllib.request.Request(
base + "/topics/" + topic_path + "/messages",
data=str(body).encode("utf-8"),
method="POST",
)
urllib.request.urlopen(_send, timeout=2).read()
_records_raw = urllib.request.urlopen(
base + "/topics/" + topic_path + "/records?max=1",
timeout=2,
).read()
_records = json.loads(_records_raw.decode("utf-8") or "{{}}").get("records", [])
if not _records:
return False
for _rec in _records:
_h(_rec.get("value", ""))
_offset = str(_rec.get("offset", "0"))
_commit = urllib.request.Request(
base + "/topics/" + topic_path + "/commit",
data=urllib.parse.urlencode({{"offset": _offset}}).encode("utf-8"),
method="POST",
)
urllib.request.urlopen(_commit, timeout=2).read()
return True
except SystemExit:
raise
except Exception as _e:
print(f"NYX_KAFKA_HTTP_FALLBACK: {{type(_e).__name__}}: {{_e}}", file=sys.stderr, flush=True)
return False
def _nyx_try_real_sqs(queue, body, handler_name):
endpoint = os.environ.get("NYX_SQS_ENDPOINT", "")
if not (endpoint.startswith("http://") or endpoint.startswith("https://")):
@ -1131,6 +1176,7 @@ except Exception as _e:
pubsub_src = pubsub_src,
rabbit_src = rabbit_src,
register_and_publish = indent_lines(&register_and_publish, " "),
kafka_publish_marker = crate::dynamic::stubs::KAFKA_PUBLISH_MARKER,
sqs_publish_marker = crate::dynamic::stubs::SQS_PUBLISH_MARKER,
);
HarnessSource {

View file

@ -951,10 +951,34 @@ def __nyx_try_execute_migration_sqlite(value)
end
end
def __nyx_patch_active_record_sql_recording
return unless defined?(ActiveRecord::Base)
return unless ActiveRecord::Base.respond_to?(:connection)
conn = ActiveRecord::Base.connection
return if conn.instance_variable_defined?(:@__nyx_sql_recording_patched)
conn.instance_variable_set(:@__nyx_sql_recording_patched, true)
if conn.respond_to?(:execute)
original_execute = conn.method(:execute)
conn.define_singleton_method(:execute) do |sql, *args, &blk|
__nyx_record_migration_result(sql, 'active_record')
original_execute.call(sql, *args, &blk)
end
end
end
# ActiveRecord migrations expose `up` / `down` / `change` on a subclass.
if Object.const_defined?({handler:?})
cls = Object.const_get({handler:?})
begin
if defined?(ActiveRecord::Migration) && cls.is_a?(Class) && cls < ActiveRecord::Migration
begin
__nyx_patch_active_record_sql_recording
cls.migrate(:up)
exit 0
rescue StandardError => e
STDERR.puts("NYX_ACTIVE_RECORD_MIGRATION_FALLBACK: #{{e.class.name}}: #{{e.message}}")
end
end
inst = cls.new
if inst.respond_to?(:table_name=)
begin

View file

@ -898,8 +898,10 @@ fn rewrite_extra_env_for_container(
{
return (k.clone(), format!("{}/{idx}", docker::STUB_MOUNT_ROOT));
}
if matches!(k.as_str(), "NYX_HTTP_ENDPOINT" | "NYX_SQS_ENDPOINT")
&& let Some(rest) = v.strip_prefix("http://127.0.0.1:")
if matches!(
k.as_str(),
"NYX_HTTP_ENDPOINT" | "NYX_KAFKA_ENDPOINT" | "NYX_SQS_ENDPOINT"
) && let Some(rest) = v.strip_prefix("http://127.0.0.1:")
{
return (k.clone(), format!("http://host-gateway:{rest}"));
}
@ -2276,6 +2278,10 @@ mod tests {
"NYX_HTTP_ENDPOINT".to_owned(),
"http://127.0.0.1:12345".to_owned(),
),
(
"NYX_KAFKA_ENDPOINT".to_owned(),
"http://127.0.0.1:22334/topics".to_owned(),
),
(
"NYX_SQS_ENDPOINT".to_owned(),
"http://127.0.0.1:23456/jobs".to_owned(),
@ -2289,6 +2295,10 @@ mod tests {
"NYX_HTTP_ENDPOINT".to_owned(),
"http://host-gateway:12345".to_owned(),
),
(
"NYX_KAFKA_ENDPOINT".to_owned(),
"http://host-gateway:22334/topics".to_owned(),
),
(
"NYX_SQS_ENDPOINT".to_owned(),
"http://host-gateway:23456/jobs".to_owned(),

View file

@ -30,6 +30,7 @@ pub struct BrokerStub {
tempdir: Option<TempDir>,
log_path: PathBuf,
cursor: Mutex<u64>,
kafka_listener: Option<KafkaListener>,
sqs_listener: Option<SqsListener>,
}
@ -42,6 +43,11 @@ impl BrokerStub {
.path()
.join(format!("nyx_{}_stub.events.log", kind.tag()));
std::fs::File::create(&log_path)?;
let kafka_listener = if kind == StubKind::Kafka {
start_kafka_listener(log_path.clone())?
} else {
None
};
let sqs_listener = if kind == StubKind::Sqs {
start_sqs_listener(log_path.clone())?
} else {
@ -52,6 +58,7 @@ impl BrokerStub {
tempdir: Some(tempdir),
log_path,
cursor: Mutex::new(0),
kafka_listener,
sqs_listener,
})
}
@ -107,6 +114,9 @@ impl StubProvider for BrokerStub {
}
fn endpoint(&self) -> String {
if let Some(listener) = &self.kafka_listener {
return format!("http://127.0.0.1:{}", listener.port);
}
if let Some(listener) = &self.sqs_listener {
return format!("http://127.0.0.1:{}", listener.port);
}
@ -182,6 +192,10 @@ fn parse_broker_log_line(line: &str) -> (&str, &str, &str) {
impl Drop for BrokerStub {
fn drop(&mut self) {
if let Some(listener) = &self.kafka_listener {
listener.shutdown.store(true, Ordering::Relaxed);
let _ = TcpStream::connect(format!("127.0.0.1:{}", listener.port));
}
if let Some(listener) = &self.sqs_listener {
listener.shutdown.store(true, Ordering::Relaxed);
let _ = TcpStream::connect(format!("127.0.0.1:{}", listener.port));
@ -190,6 +204,159 @@ impl Drop for BrokerStub {
}
}
#[derive(Debug)]
struct KafkaListener {
port: u16,
shutdown: Arc<AtomicBool>,
}
#[derive(Debug, Clone)]
struct KafkaMessage {
offset: u64,
value: String,
}
#[derive(Debug, Default)]
struct KafkaState {
next_offsets: BTreeMap<String, u64>,
topics: BTreeMap<String, VecDeque<KafkaMessage>>,
inflight: BTreeMap<(String, u64), KafkaMessage>,
}
fn start_kafka_listener(log_path: PathBuf) -> std::io::Result<Option<KafkaListener>> {
let listener = match TcpListener::bind("127.0.0.1:0") {
Ok(listener) => listener,
Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => return Ok(None),
Err(e) => return Err(e),
};
let port = listener.local_addr()?.port();
let shutdown = Arc::new(AtomicBool::new(false));
let state = Arc::new(Mutex::new(KafkaState::default()));
let shutdown_clone = Arc::clone(&shutdown);
let state_clone = Arc::clone(&state);
std::thread::spawn(move || kafka_accept_loop(listener, shutdown_clone, state_clone, log_path));
Ok(Some(KafkaListener { port, shutdown }))
}
fn kafka_accept_loop(
listener: TcpListener,
shutdown: Arc<AtomicBool>,
state: Arc<Mutex<KafkaState>>,
log_path: PathBuf,
) {
for stream in listener.incoming() {
if shutdown.load(Ordering::Relaxed) {
break;
}
let Ok(stream) = stream else { continue };
let _ = stream.set_read_timeout(Some(Duration::from_secs(2)));
let _ = stream.set_write_timeout(Some(Duration::from_secs(2)));
let state = Arc::clone(&state);
let log_path = log_path.clone();
std::thread::spawn(move || handle_kafka_connection(stream, state, &log_path));
}
}
fn handle_kafka_connection(mut stream: TcpStream, state: Arc<Mutex<KafkaState>>, log_path: &Path) {
let Some(req) = read_http_request(&stream) else {
return;
};
let response = match handle_kafka_request(&req, state, log_path) {
Ok(body) => http_response_with_type(200, "OK", "application/json", &body),
Err(body) => http_response_with_type(400, "Bad Request", "application/json", &body),
};
let _ = stream.write_all(response.as_bytes());
}
fn handle_kafka_request(
req: &HttpRequest,
state: Arc<Mutex<KafkaState>>,
log_path: &Path,
) -> Result<String, String> {
let Some((topic, action)) = kafka_path_parts(&req.path) else {
return Err(json_error("invalid kafka stub path"));
};
match action.as_str() {
"messages" => {
let mut guard = state.lock().map_err(|_| json_error("internal error"))?;
let offset = guard.next_offsets.entry(topic.clone()).or_insert(0);
let message = KafkaMessage {
offset: *offset,
value: req.body.clone(),
};
*offset += 1;
guard
.topics
.entry(topic.clone())
.or_default()
.push_back(message.clone());
let _ = append_broker_event(log_path, "publish", &topic, &message.value);
Ok(serde_json::json!({
"topic": topic,
"offset": message.offset
})
.to_string())
}
"records" => {
let params = parse_form(&req.query);
let max_records = params
.get("max")
.and_then(|v| v.parse::<usize>().ok())
.unwrap_or(1)
.clamp(1, 100);
let mut guard = state.lock().map_err(|_| json_error("internal error"))?;
let mut records = Vec::new();
for _ in 0..max_records {
let Some(message) = guard.topics.entry(topic.clone()).or_default().pop_front()
else {
break;
};
let _ = append_broker_event(log_path, "deliver", &topic, &message.value);
guard
.inflight
.insert((topic.clone(), message.offset), message.clone());
records.push(serde_json::json!({
"topic": topic,
"offset": message.offset,
"value": message.value
}));
}
Ok(serde_json::json!({ "records": records }).to_string())
}
"commit" => {
let params = parse_form(&req.body);
let offset = params
.get("offset")
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(0);
if let Ok(mut guard) = state.lock()
&& guard.inflight.remove(&(topic.clone(), offset)).is_some()
{
let _ = append_broker_event(log_path, "ack", &topic, &offset.to_string());
}
Ok(serde_json::json!({ "committed": true }).to_string())
}
_ => Err(json_error("invalid kafka stub action")),
}
}
fn kafka_path_parts(path: &str) -> Option<(String, String)> {
let mut parts = path.trim_matches('/').split('/');
if parts.next()? != "topics" {
return None;
}
let topic = parts.next().map(percent_decode)?;
let action = parts.next()?.to_owned();
if topic.is_empty() || parts.next().is_some() {
return None;
}
Some((topic, action))
}
fn json_error(message: &str) -> String {
serde_json::json!({ "error": message }).to_string()
}
#[derive(Debug)]
struct SqsListener {
port: u16,
@ -427,8 +594,12 @@ fn handle_sqs_request(
}
fn http_response(status: u16, reason: &str, body: &str) -> String {
http_response_with_type(status, reason, "text/xml", body)
}
fn http_response_with_type(status: u16, reason: &str, content_type: &str, body: &str) -> String {
format!(
"HTTP/1.1 {status} {reason}\r\ncontent-type: text/xml\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
"HTTP/1.1 {status} {reason}\r\ncontent-type: {content_type}\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
body.len()
)
}
@ -544,7 +715,11 @@ mod tests {
let dir = TempDir::new().unwrap();
let stub = BrokerStub::start(StubKind::Kafka, dir.path()).unwrap();
assert!(stub.log_path().exists());
assert_eq!(stub.endpoint(), "loopback://kafka");
let endpoint = stub.endpoint();
assert!(
endpoint == "loopback://kafka" || endpoint.starts_with("http://127.0.0.1:"),
"Kafka endpoint should be loopback fallback or HTTP emulator, got {endpoint}"
);
assert_eq!(
stub.recording_endpoint().unwrap().0,
StubKind::Kafka.broker_log_env_var().unwrap()
@ -580,6 +755,52 @@ mod tests {
);
}
#[test]
fn kafka_broker_exposes_http_emulator() {
let dir = TempDir::new().unwrap();
let stub = BrokerStub::start(StubKind::Kafka, dir.path()).unwrap();
let endpoint = stub.endpoint();
if endpoint == "loopback://kafka" {
return;
}
assert!(
endpoint.starts_with("http://127.0.0.1:"),
"Kafka endpoint should be a host-side HTTP emulator, got {endpoint}"
);
}
#[test]
fn kafka_http_emulator_records_publish_deliver_ack() {
let dir = TempDir::new().unwrap();
let stub = BrokerStub::start(StubKind::Kafka, dir.path()).unwrap();
let endpoint = stub.endpoint();
if endpoint == "loopback://kafka" {
return;
}
let port: u16 = endpoint
.trim_start_matches("http://127.0.0.1:")
.parse()
.unwrap();
let send = http_post(port, "/topics/orders/messages", "NYX\tPAYLOAD");
assert!(send.contains(r#""offset":0"#), "{send}");
let receive = http_get(port, "/topics/orders/records?max=1");
assert!(receive.contains(r#""value":"NYX\tPAYLOAD""#), "{receive}");
let commit = http_post(port, "/topics/orders/commit", "offset=0");
assert!(commit.contains(r#""committed":true"#), "{commit}");
let events = stub.drain_events();
let actions: Vec<&str> = events
.iter()
.map(|ev| ev.detail.get("action").unwrap().as_str())
.collect();
assert_eq!(actions, vec!["publish", "deliver", "ack"]);
assert_eq!(events[0].detail.get("destination").unwrap(), "orders");
assert_eq!(events[1].detail.get("payload").unwrap(), "NYX\tPAYLOAD");
assert_eq!(events[2].detail.get("payload").unwrap(), "0");
}
#[test]
fn sqs_query_emulator_records_publish_deliver_ack() {
let dir = TempDir::new().unwrap();
@ -669,6 +890,16 @@ mod tests {
out
}
fn http_get(port: u16, path: &str) -> String {
let mut s = TcpStream::connect(format!("127.0.0.1:{port}")).unwrap();
let req =
format!("GET {path} HTTP/1.1\r\nhost: 127.0.0.1:{port}\r\nconnection: close\r\n\r\n");
s.write_all(req.as_bytes()).unwrap();
let mut out = String::new();
s.read_to_string(&mut out).unwrap();
out
}
fn form_escape(input: &str) -> String {
let mut out = String::new();
for b in input.bytes() {

View file

@ -176,6 +176,8 @@ fn message_handler_python_dispatch_subscribes_to_loopback() {
entry_file("kafka_python"),
);
let h = lang::emit(&spec).expect("emit ok");
assert!(h.source.contains("_nyx_try_kafka_http"));
assert!(h.source.contains("NYX_KAFKA_ENDPOINT"));
assert!(h.source.contains("NyxKafkaLoopback"));
assert!(h.source.contains("subscribe"));
assert!(h.source.contains("poll"));
@ -192,6 +194,11 @@ fn message_handler_python_dispatch_subscribes_to_loopback() {
fn message_handler_java_emits_reflective_dispatch() {
let spec = make_spec(Lang::Java, "orders", "onMessage", entry_file("kafka_java"));
let h = lang::emit(&spec).expect("emit ok");
assert!(h.source.contains("nyxTryRealKafkaClient"));
assert!(h.source.contains("MockConsumer"));
assert!(h.source.contains("commitSync"));
assert!(h.source.contains("nyxTryKafkaHttp"));
assert!(h.source.contains("NYX_KAFKA_ENDPOINT"));
assert!(h.source.contains("NyxKafkaLoopback"));
assert!(h.source.contains("Class.forName"));
assert!(h.source.contains("getDeclaredMethod"));

View file

@ -945,6 +945,7 @@ fn migration_ruby_harness_carries_sentinel_and_handler() {
assert!(h.source.contains("AddIndex"));
assert!(h.source.contains("__nyx_stub_sql_record"));
assert!(h.source.contains("ActiveRecord::Base.establish_connection"));
assert!(h.source.contains("cls.migrate(:up)"));
assert!(h.source.contains("SQLite3::Database"));
assert!(h.source.contains("NYX_SQL_ENDPOINT"));
}