apunkt/nyx
1
0
Fork 0
mirror of https://github.com/elicpeter/nyx.git synced 2026-06-24 20:28:06 +02:00
Code Issues Projects Releases Packages Wiki Activity

**refactor(dynamic): add Kafka HTTP emulator with publish/poll/commit support, extend endpoint rewriting and stub event recording across Java, Python, and Rust**

Browse source
This commit is contained in:
elipeter 2026-05-27 11:01:46 -05:00
parent 433036aead
commit 57d3677bd4
7 changed files with 564 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

245
src/dynamic/lang/java.rs
View file

@ -3873,26 +3873,29 @@ fn emit_message_handler_harness(
JavaBroker::Kafka => (
crate::dynamic::stubs::KAFKA_PUBLISH_MARKER,
format!(
r#" NyxKafkaLoopback brokerRef = new NyxKafkaLoopback();
System.out.println({publish_marker:?} + " " + {queue:?});
nyxRecordBrokerPublish("NYX_KAFKA_LOG", {queue:?}, payload);
brokerRef.publish({queue:?}, payload);
for (NyxKafkaRecord rec : brokerRef.poll({queue:?}, 1)) {{
nyxRecordBrokerEvent("NYX_KAFKA_LOG", "deliver", {queue:?}, rec.value);
System.out.println("__NYX_SINK_HIT__");
boolean success = false;
try {{
java.lang.reflect.Method m = entryInst.getClass().getDeclaredMethod({handler:?}, String.class);
m.setAccessible(true);
m.invoke(entryInst, rec.value);
success = true;
}} catch (Exception e) {{
Throwable c = (e instanceof java.lang.reflect.InvocationTargetException && e.getCause() != null) ? e.getCause() : e;
System.err.println("NYX_EXCEPTION: " + c.getClass().getName() + ": " + c.getMessage());
}}
if (success) {{
brokerRef.commit(rec);
nyxRecordBrokerEvent("NYX_KAFKA_LOG", "ack", {queue:?}, Long.toString(rec.offset));
r#" if (!nyxTryRealKafkaClient({queue:?}, payload, entryInst, {handler:?})
&& !nyxTryKafkaHttp({queue:?}, payload, entryInst, {handler:?})) {{
NyxKafkaLoopback brokerRef = new NyxKafkaLoopback();
System.out.println({publish_marker:?} + " " + {queue:?});
nyxRecordBrokerPublish("NYX_KAFKA_LOG", {queue:?}, payload);
brokerRef.publish({queue:?}, payload);
for (NyxKafkaRecord rec : brokerRef.poll({queue:?}, 1)) {{
nyxRecordBrokerEvent("NYX_KAFKA_LOG", "deliver", {queue:?}, rec.value);
System.out.println("__NYX_SINK_HIT__");
boolean success = false;
try {{
java.lang.reflect.Method m = entryInst.getClass().getDeclaredMethod({handler:?}, String.class);
m.setAccessible(true);
m.invoke(entryInst, rec.value);
success = true;
}} catch (Exception e) {{
Throwable c = (e instanceof java.lang.reflect.InvocationTargetException && e.getCause() != null) ? e.getCause() : e;
System.err.println("NYX_EXCEPTION: " + c.getClass().getName() + ": " + c.getMessage());
}}
if (success) {{
brokerRef.commit(rec);
nyxRecordBrokerEvent("NYX_KAFKA_LOG", "ack", {queue:?}, Long.toString(rec.offset));
}}
}}
}}"#,
handler = handler,
@ -3928,6 +3931,207 @@ public class NyxHarness {{
}}
}}
static boolean nyxTryKafkaHttp(String topic, String payload, Object entryInst, String handler) {{
String endpoint = System.getenv("NYX_KAFKA_ENDPOINT");
if (endpoint == null || !(endpoint.startsWith("http://") || endpoint.startsWith("https://"))) {{
return false;
}}
try {{
String base = endpoint.replaceAll("/+$", "");
String topicPath = java.net.URLEncoder.encode(topic, java.nio.charset.StandardCharsets.UTF_8);
System.out.println({kafka_publish_marker:?} + " " + topic);
nyxHttpRequest(
"POST",
base + "/topics/" + topicPath + "/messages",
payload.getBytes(java.nio.charset.StandardCharsets.UTF_8)
);
String recordsJson = nyxHttpRequest(
"GET",
base + "/topics/" + topicPath + "/records?max=1",
new byte[0]
);
if (recordsJson == null || !recordsJson.contains("\"records\"") || !recordsJson.contains("\"value\"")) {{
return false;
}}
String value = nyxJsonStringField(recordsJson, "value");
String offset = nyxJsonNumberField(recordsJson, "offset");
if (offset == null || offset.isEmpty()) {{
offset = "0";
}}
System.out.println("__NYX_SINK_HIT__");
boolean success = false;
try {{
java.lang.reflect.Method m = entryInst.getClass().getDeclaredMethod(handler, String.class);
m.setAccessible(true);
m.invoke(entryInst, value);
success = true;
}} catch (Exception e) {{
Throwable c = (e instanceof java.lang.reflect.InvocationTargetException && e.getCause() != null) ? e.getCause() : e;
System.err.println("NYX_EXCEPTION: " + c.getClass().getName() + ": " + c.getMessage());
}}
if (success) {{
String body = "offset=" + java.net.URLEncoder.encode(offset, java.nio.charset.StandardCharsets.UTF_8);
nyxHttpRequest(
"POST",
base + "/topics/" + topicPath + "/commit",
body.getBytes(java.nio.charset.StandardCharsets.UTF_8)
);
}}
return true;
}} catch (Throwable e) {{
System.err.println("NYX_KAFKA_HTTP_FALLBACK: " + e.getClass().getName() + ": " + e.getMessage());
return false;
}}
}}
static boolean nyxTryRealKafkaClient(String topic, String payload, Object entryInst, String handler) {{
Object consumer = null;
try {{
Class<?> mockConsumerClass = Class.forName("org.apache.kafka.clients.consumer.MockConsumer");
Class<?> resetClass = Class.forName("org.apache.kafka.clients.consumer.OffsetResetStrategy");
Object earliest = java.lang.Enum.valueOf(resetClass.asSubclass(java.lang.Enum.class), "EARLIEST");
consumer = mockConsumerClass.getConstructor(resetClass).newInstance(earliest);
Class<?> topicPartitionClass = Class.forName("org.apache.kafka.common.TopicPartition");
Object partition = topicPartitionClass.getConstructor(String.class, int.class).newInstance(topic, 0);
java.util.List<Object> partitions = java.util.Collections.singletonList(partition);
mockConsumerClass.getMethod("subscribe", java.util.Collection.class)
.invoke(consumer, java.util.Collections.singletonList(topic));
mockConsumerClass.getMethod("rebalance", java.util.Collection.class).invoke(consumer, partitions);
java.util.Map<Object, Long> beginnings = new java.util.HashMap<>();
beginnings.put(partition, Long.valueOf(0L));
mockConsumerClass.getMethod("updateBeginningOffsets", java.util.Map.class).invoke(consumer, beginnings);
Class<?> recordClass = Class.forName("org.apache.kafka.clients.consumer.ConsumerRecord");
Object record = null;
for (java.lang.reflect.Constructor<?> ctor : recordClass.getConstructors()) {{
if (ctor.getParameterCount() == 5) {{
record = ctor.newInstance(topic, Integer.valueOf(0), Long.valueOf(0L), null, payload);
break;
}}
}}
if (record == null) {{
return false;
}}
System.out.println({kafka_publish_marker:?} + " " + topic);
nyxRecordBrokerPublish("NYX_KAFKA_LOG", topic, payload);
mockConsumerClass.getMethod("addRecord", recordClass).invoke(consumer, record);
Object records = mockConsumerClass.getMethod("poll", java.time.Duration.class)
.invoke(consumer, java.time.Duration.ofMillis(10));
if (!(records instanceof Iterable)) {{
return false;
}}
boolean delivered = false;
for (Object rec : (Iterable<?>) records) {{
String value = String.valueOf(rec.getClass().getMethod("value").invoke(rec));
long offset = ((Number) rec.getClass().getMethod("offset").invoke(rec)).longValue();
nyxRecordBrokerEvent("NYX_KAFKA_LOG", "deliver", topic, value);
System.out.println("__NYX_SINK_HIT__");
boolean success = false;
try {{
java.lang.reflect.Method m = entryInst.getClass().getDeclaredMethod(handler, String.class);
m.setAccessible(true);
m.invoke(entryInst, value);
success = true;
}} catch (Exception e) {{
Throwable c = (e instanceof java.lang.reflect.InvocationTargetException && e.getCause() != null) ? e.getCause() : e;
System.err.println("NYX_EXCEPTION: " + c.getClass().getName() + ": " + c.getMessage());
}}
if (success) {{
Class<?> offsetClass = Class.forName("org.apache.kafka.clients.consumer.OffsetAndMetadata");
Object metadata = offsetClass.getConstructor(long.class).newInstance(Long.valueOf(offset + 1L));
java.util.Map<Object, Object> commits = new java.util.HashMap<>();
commits.put(partition, metadata);
mockConsumerClass.getMethod("commitSync", java.util.Map.class).invoke(consumer, commits);
nyxRecordBrokerEvent("NYX_KAFKA_LOG", "ack", topic, Long.toString(offset));
}}
delivered = true;
}}
return delivered;
}} catch (ClassNotFoundException missingKafkaClient) {{
return false;
}} catch (Throwable e) {{
System.err.println("NYX_REAL_KAFKA_FALLBACK: " + e.getClass().getName() + ": " + e.getMessage());
return false;
}} finally {{
if (consumer instanceof AutoCloseable) {{
try {{
((AutoCloseable) consumer).close();
}} catch (Exception ignored) {{
}}
}}
}}
}}
static String nyxHttpRequest(String method, String target, byte[] body) throws Exception {{
java.net.HttpURLConnection conn = (java.net.HttpURLConnection) java.net.URI.create(target).toURL().openConnection();
conn.setRequestMethod(method);
conn.setConnectTimeout(1000);
conn.setReadTimeout(2000);
if (body != null && body.length > 0) {{
conn.setDoOutput(true);
conn.setRequestProperty("Content-Type", "application/octet-stream");
conn.setRequestProperty("Content-Length", Integer.toString(body.length));
try (java.io.OutputStream os = conn.getOutputStream()) {{
os.write(body);
}}
}}
java.io.InputStream is = conn.getResponseCode() >= 400 ? conn.getErrorStream() : conn.getInputStream();
if (is == null) {{
return "";
}}
try (java.io.InputStream input = is) {{
byte[] data = input.readAllBytes();
return new String(data, java.nio.charset.StandardCharsets.UTF_8);
}} finally {{
conn.disconnect();
}}
}}
static String nyxJsonStringField(String json, String field) {{
String needle = "\"" + field + "\":\"";
int start = json.indexOf(needle);
if (start < 0) return "";
start += needle.length();
StringBuilder out = new StringBuilder();
boolean escaped = false;
for (int i = start; i < json.length(); i++) {{
char ch = json.charAt(i);
if (escaped) {{
switch (ch) {{
case 'n': out.append('\n'); break;
case 'r': out.append('\r'); break;
case 't': out.append('\t'); break;
case '"': out.append('"'); break;
case '\\': out.append('\\'); break;
default: out.append(ch); break;
}}
escaped = false;
}} else if (ch == '\\') {{
escaped = true;
}} else if (ch == '"') {{
break;
}} else {{
out.append(ch);
}}
}}
return out.toString();
}}
static String nyxJsonNumberField(String json, String field) {{
String needle = "\"" + field + "\":";
int start = json.indexOf(needle);
if (start < 0) return "";
start += needle.length();
int end = start;
while (end < json.length() && Character.isDigit(json.charAt(end))) {{
end++;
}}
return json.substring(start, end);
}}
static boolean nyxTryRealSqs(String queue, String payload, Object entryInst, String handler) {{
String endpoint = System.getenv("NYX_SQS_ENDPOINT");
if (endpoint == null || !(endpoint.startsWith("http://") || endpoint.startsWith("https://"))) {{
@ -4052,6 +4256,7 @@ public class NyxHarness {{
"#,
entry_class = entry_class,
dispatch_block = dispatch_block,
kafka_publish_marker = crate::dynamic::stubs::KAFKA_PUBLISH_MARKER,
sqs_publish_marker = crate::dynamic::stubs::SQS_PUBLISH_MARKER,
);
HarnessSource {

78
src/dynamic/lang/python.rs
View file

@ -1018,22 +1018,23 @@ _chan.basic_publish(exchange="", routing_key={queue:?}, body=payload)"#,
publish_marker = crate::dynamic::stubs::RABBIT_PUBLISH_MARKER,
),
PythonBroker::Kafka => format!(
r#"_loop = NyxKafkaLoopback()
def _nyx_kafka_dispatch(message):
_h = getattr(_entry_mod, {handler:?}, None)
if _h is None:
print("NYX_HANDLER_NOT_FOUND: " + {handler:?}, file=sys.stderr, flush=True)
sys.exit(78)
_h(message)
_loop.subscribe({queue:?}, _nyx_kafka_dispatch)
print({publish_marker:?} + " " + {queue:?}, flush=True)
_nyx_record_broker_publish("NYX_KAFKA_LOG", {queue:?}, payload)
_loop.publish({queue:?}, payload)
for _record in _loop.poll({queue:?}, max_records=1):
_nyx_record_broker_event("NYX_KAFKA_LOG", "deliver", {queue:?}, _record.value)
_nyx_kafka_dispatch(_record.value)
_loop.commit(_record)
_nyx_record_broker_event("NYX_KAFKA_LOG", "ack", {queue:?}, str(_record.offset))"#,
r#"if not _nyx_try_kafka_http({queue:?}, payload, {handler:?}):
_loop = NyxKafkaLoopback()
def _nyx_kafka_dispatch(message):
_h = getattr(_entry_mod, {handler:?}, None)
if _h is None:
print("NYX_HANDLER_NOT_FOUND: " + {handler:?}, file=sys.stderr, flush=True)
sys.exit(78)
_h(message)
_loop.subscribe({queue:?}, _nyx_kafka_dispatch)
print({publish_marker:?} + " " + {queue:?}, flush=True)
_nyx_record_broker_publish("NYX_KAFKA_LOG", {queue:?}, payload)
_loop.publish({queue:?}, payload)
for _record in _loop.poll({queue:?}, max_records=1):
_nyx_record_broker_event("NYX_KAFKA_LOG", "deliver", {queue:?}, _record.value)
_nyx_kafka_dispatch(_record.value)
_loop.commit(_record)
_nyx_record_broker_event("NYX_KAFKA_LOG", "ack", {queue:?}, str(_record.offset))"#,
handler = handler,
queue = queue,
publish_marker = crate::dynamic::stubs::KAFKA_PUBLISH_MARKER,
@ -1064,6 +1065,50 @@ def _nyx_record_broker_event(env_name, action, destination, body):
def _nyx_record_broker_publish(env_name, destination, body):
_nyx_record_broker_event(env_name, "publish", destination, body)
def _nyx_try_kafka_http(topic, body, handler_name):
endpoint = os.environ.get("NYX_KAFKA_ENDPOINT", "")
if not (endpoint.startswith("http://") or endpoint.startswith("https://")):
return False
_h = getattr(_entry_mod, handler_name, None)
if _h is None:
print("NYX_HANDLER_NOT_FOUND: " + handler_name, file=sys.stderr, flush=True)
sys.exit(78)
try:
import json
import urllib.parse
import urllib.request
base = endpoint.rstrip("/")
topic_path = urllib.parse.quote(str(topic), safe="")
print({kafka_publish_marker:?} + " " + str(topic), flush=True)
_send = urllib.request.Request(
base + "/topics/" + topic_path + "/messages",
data=str(body).encode("utf-8"),
method="POST",
)
urllib.request.urlopen(_send, timeout=2).read()
_records_raw = urllib.request.urlopen(
base + "/topics/" + topic_path + "/records?max=1",
timeout=2,
).read()
_records = json.loads(_records_raw.decode("utf-8") or "{{}}").get("records", [])
if not _records:
return False
for _rec in _records:
_h(_rec.get("value", ""))
_offset = str(_rec.get("offset", "0"))
_commit = urllib.request.Request(
base + "/topics/" + topic_path + "/commit",
data=urllib.parse.urlencode({{"offset": _offset}}).encode("utf-8"),
method="POST",
)
urllib.request.urlopen(_commit, timeout=2).read()
return True
except SystemExit:
raise
except Exception as _e:
print(f"NYX_KAFKA_HTTP_FALLBACK: {{type(_e).__name__}}: {{_e}}", file=sys.stderr, flush=True)
return False
def _nyx_try_real_sqs(queue, body, handler_name):
endpoint = os.environ.get("NYX_SQS_ENDPOINT", "")
if not (endpoint.startswith("http://") or endpoint.startswith("https://")):
@ -1131,6 +1176,7 @@ except Exception as _e:
pubsub_src = pubsub_src,
rabbit_src = rabbit_src,
register_and_publish = indent_lines(&register_and_publish, " "),
kafka_publish_marker = crate::dynamic::stubs::KAFKA_PUBLISH_MARKER,
sqs_publish_marker = crate::dynamic::stubs::SQS_PUBLISH_MARKER,
);
HarnessSource {

24
src/dynamic/lang/ruby.rs
View file

@ -951,10 +951,34 @@ def __nyx_try_execute_migration_sqlite(value)
end
end
def __nyx_patch_active_record_sql_recording
return unless defined?(ActiveRecord::Base)
return unless ActiveRecord::Base.respond_to?(:connection)
conn = ActiveRecord::Base.connection
return if conn.instance_variable_defined?(:@__nyx_sql_recording_patched)
conn.instance_variable_set(:@__nyx_sql_recording_patched, true)
if conn.respond_to?(:execute)
original_execute = conn.method(:execute)
conn.define_singleton_method(:execute) do |sql, *args, &blk|
__nyx_record_migration_result(sql, 'active_record')
original_execute.call(sql, *args, &blk)
end
end
end
# ActiveRecord migrations expose `up` / `down` / `change` on a subclass.
if Object.const_defined?({handler:?})
cls = Object.const_get({handler:?})
begin
if defined?(ActiveRecord::Migration) && cls.is_a?(Class) && cls < ActiveRecord::Migration
begin
__nyx_patch_active_record_sql_recording
cls.migrate(:up)
exit 0
rescue StandardError => e
STDERR.puts("NYX_ACTIVE_RECORD_MIGRATION_FALLBACK: #{{e.class.name}}: #{{e.message}}")
end
end
inst = cls.new
if inst.respond_to?(:table_name=)
begin