mirror of
https://github.com/elicpeter/nyx.git
synced 2026-06-24 20:28:06 +02:00
refactor(dynamic): introduce publish/poll/commit cycle for Kafka, expand SQS loopback with receive/delete support, enhance event recording, and unify migration SQL handling across frameworks
This commit is contained in:
elipeter
parent
ed96f94bb5
commit
6ee2bdda36
11 changed files with 515 additions and 71 deletions
|
|
@ -3804,20 +3804,26 @@ fn emit_message_handler_harness(
|
|||
crate::dynamic::stubs::SQS_PUBLISH_MARKER,
|
||||
format!(
|
||||
r#" NyxSqsLoopback brokerRef = new NyxSqsLoopback();
|
||||
brokerRef.subscribe({queue:?}, env -> {{
|
||||
System.out.println({publish_marker:?} + " " + {queue:?});
|
||||
nyxRecordBrokerPublish("NYX_SQS_LOG", {queue:?}, payload);
|
||||
brokerRef.publish({queue:?}, payload);
|
||||
for (java.util.Map<String, String> env : brokerRef.receiveMessage({queue:?}, 1)) {{
|
||||
nyxRecordBrokerEvent("NYX_SQS_LOG", "deliver", {queue:?}, env.getOrDefault("Body", ""));
|
||||
System.out.println("__NYX_SINK_HIT__");
|
||||
boolean success = false;
|
||||
try {{
|
||||
java.lang.reflect.Method m = entryInst.getClass().getDeclaredMethod({handler:?}, java.util.Map.class);
|
||||
m.setAccessible(true);
|
||||
m.invoke(entryInst, env);
|
||||
success = true;
|
||||
}} catch (Exception e) {{
|
||||
Throwable c = (e instanceof java.lang.reflect.InvocationTargetException && e.getCause() != null) ? e.getCause() : e;
|
||||
System.err.println("NYX_EXCEPTION: " + c.getClass().getName() + ": " + c.getMessage());
|
||||
}}
|
||||
}});
|
||||
System.out.println({publish_marker:?} + " " + {queue:?});
|
||||
nyxRecordBrokerPublish("NYX_SQS_LOG", {queue:?}, payload);
|
||||
brokerRef.publish({queue:?}, payload);"#,
|
||||
if (success && brokerRef.deleteMessage({queue:?}, env.getOrDefault("ReceiptHandle", ""))) {{
|
||||
nyxRecordBrokerEvent("NYX_SQS_LOG", "ack", {queue:?}, env.getOrDefault("ReceiptHandle", ""));
|
||||
}}
|
||||
}}"#,
|
||||
handler = handler,
|
||||
queue = queue,
|
||||
publish_marker = crate::dynamic::stubs::SQS_PUBLISH_MARKER,
|
||||
|
|
@ -3859,20 +3865,27 @@ fn emit_message_handler_harness(
|
|||
crate::dynamic::stubs::KAFKA_PUBLISH_MARKER,
|
||||
format!(
|
||||
r#" NyxKafkaLoopback brokerRef = new NyxKafkaLoopback();
|
||||
brokerRef.subscribe({queue:?}, body -> {{
|
||||
System.out.println({publish_marker:?} + " " + {queue:?});
|
||||
nyxRecordBrokerPublish("NYX_KAFKA_LOG", {queue:?}, payload);
|
||||
brokerRef.publish({queue:?}, payload);
|
||||
for (NyxKafkaRecord rec : brokerRef.poll({queue:?}, 1)) {{
|
||||
nyxRecordBrokerEvent("NYX_KAFKA_LOG", "deliver", {queue:?}, rec.value);
|
||||
System.out.println("__NYX_SINK_HIT__");
|
||||
boolean success = false;
|
||||
try {{
|
||||
java.lang.reflect.Method m = entryInst.getClass().getDeclaredMethod({handler:?}, String.class);
|
||||
m.setAccessible(true);
|
||||
m.invoke(entryInst, body);
|
||||
m.invoke(entryInst, rec.value);
|
||||
success = true;
|
||||
}} catch (Exception e) {{
|
||||
Throwable c = (e instanceof java.lang.reflect.InvocationTargetException && e.getCause() != null) ? e.getCause() : e;
|
||||
System.err.println("NYX_EXCEPTION: " + c.getClass().getName() + ": " + c.getMessage());
|
||||
}}
|
||||
}});
|
||||
System.out.println({publish_marker:?} + " " + {queue:?});
|
||||
nyxRecordBrokerPublish("NYX_KAFKA_LOG", {queue:?}, payload);
|
||||
brokerRef.publish({queue:?}, payload);"#,
|
||||
if (success) {{
|
||||
brokerRef.commit(rec);
|
||||
nyxRecordBrokerEvent("NYX_KAFKA_LOG", "ack", {queue:?}, Long.toString(rec.offset));
|
||||
}}
|
||||
}}"#,
|
||||
handler = handler,
|
||||
queue = queue,
|
||||
publish_marker = crate::dynamic::stubs::KAFKA_PUBLISH_MARKER,
|
||||
|
|
@ -3917,10 +3930,10 @@ public class NyxHarness {{
|
|||
return "";
|
||||
}}
|
||||
|
||||
static void nyxRecordBrokerPublish(String envName, String destination, String payload) {{
|
||||
static void nyxRecordBrokerEvent(String envName, String action, String destination, String payload) {{
|
||||
String path = System.getenv(envName);
|
||||
if (path == null || path.isEmpty()) return;
|
||||
String line = destination.replace('\t', ' ') + "\t" + payload + "\n";
|
||||
String line = action.replace('\t', ' ') + "\t" + destination.replace('\t', ' ') + "\t" + payload + "\n";
|
||||
try {{
|
||||
java.nio.file.Files.write(
|
||||
java.nio.file.Paths.get(path),
|
||||
|
|
@ -3931,6 +3944,10 @@ public class NyxHarness {{
|
|||
}} catch (Exception ignored) {{
|
||||
}}
|
||||
}}
|
||||
|
||||
static void nyxRecordBrokerPublish(String envName, String destination, String payload) {{
|
||||
nyxRecordBrokerEvent(envName, "publish", destination, payload);
|
||||
}}
|
||||
}}
|
||||
"#,
|
||||
entry_class = entry_class,
|
||||
|
|
|
|||
|
|
@ -928,32 +928,49 @@ if (typeof _handler !== 'function') {{
|
|||
}}
|
||||
|
||||
const _broker = new NyxSqsLoopback();
|
||||
function _nyxRecordBrokerPublish(envName, destination, body) {{
|
||||
function _nyxRecordBrokerEvent(envName, action, destination, body) {{
|
||||
const path = process.env[envName] || '';
|
||||
if (!path) return;
|
||||
try {{
|
||||
require('fs').appendFileSync(
|
||||
path,
|
||||
String(destination).replace(/\t/g, ' ') + '\t' + String(body) + '\n',
|
||||
String(action).replace(/\t/g, ' ') + '\t' +
|
||||
String(destination).replace(/\t/g, ' ') + '\t' +
|
||||
String(body) + '\n',
|
||||
'utf8'
|
||||
);
|
||||
}} catch (_) {{}}
|
||||
}}
|
||||
_broker.subscribe({queue:?}, async (envelope) => {{
|
||||
function _nyxRecordBrokerPublish(envName, destination, body) {{
|
||||
_nyxRecordBrokerEvent(envName, 'publish', destination, body);
|
||||
}}
|
||||
|
||||
async function _nyxDispatchEnvelope(envelope) {{
|
||||
try {{
|
||||
// Sink-reachability sentinel — runner's `vuln_fired && sink_hit`
|
||||
// gate requires this byte sequence on stdout / stderr.
|
||||
process.stdout.write('__NYX_SINK_HIT__\n');
|
||||
await Promise.resolve(_handler(envelope));
|
||||
return true;
|
||||
}} catch (e) {{
|
||||
process.stderr.write('NYX_EXCEPTION: ' + (e.constructor ? e.constructor.name : 'Error') + ': ' + e.message + '\n');
|
||||
return false;
|
||||
}}
|
||||
}});
|
||||
}}
|
||||
|
||||
(async () => {{
|
||||
process.stdout.write({publish_marker:?} + ' ' + {queue:?} + '\n');
|
||||
_nyxRecordBrokerPublish('NYX_SQS_LOG', {queue:?}, payload);
|
||||
_broker.publish({queue:?}, payload);
|
||||
for (const envelope of _broker.receiveMessage({queue:?}, 1)) {{
|
||||
_nyxRecordBrokerEvent('NYX_SQS_LOG', 'deliver', {queue:?}, envelope.Body || '');
|
||||
const ok = await _nyxDispatchEnvelope(envelope);
|
||||
if (ok && _broker.deleteMessage({queue:?}, envelope.ReceiptHandle || '')) {{
|
||||
_nyxRecordBrokerEvent('NYX_SQS_LOG', 'ack', {queue:?}, envelope.ReceiptHandle || '');
|
||||
}} else {{
|
||||
_broker.replayInflight();
|
||||
}}
|
||||
}}
|
||||
}})();
|
||||
"#,
|
||||
handler = handler,
|
||||
|
|
@ -1187,21 +1204,30 @@ if (_h == null) {{
|
|||
process.stderr.write('NYX_HANDLER_NOT_FOUND: ' + {handler:?} + '\n');
|
||||
process.exit(78);
|
||||
}}
|
||||
// Synthetic queryInterface for sequelize-style up/down(queryInterface, Sequelize).
|
||||
function _nyxLooksLikeSql(sql) {{
|
||||
const upper = String(sql).toUpperCase();
|
||||
return ['SELECT', 'INSERT', 'UPDATE', 'DELETE', 'CREATE', 'ALTER', 'DROP'].some((k) => upper.includes(k));
|
||||
}}
|
||||
function _nyxMigrationSqlRecord(sql, driver) {{
|
||||
if (!_nyxLooksLikeSql(sql)) return;
|
||||
__nyx_stub_sql_record(String(sql), {{ driver: driver, source: 'migration' }});
|
||||
}}
|
||||
// QueryInterface shim for sequelize-style up/down(queryInterface, Sequelize).
|
||||
const _qi = {{
|
||||
createTable: async function(){{}},
|
||||
addColumn: async function(){{}},
|
||||
dropTable: async function(){{}},
|
||||
removeColumn: async function(){{}},
|
||||
bulkInsert: async function(){{}},
|
||||
sequelize: {{ query: async function(){{}} }},
|
||||
createTable: async function(name){{ const sql = 'CREATE TABLE ' + String(name) + ' (id INTEGER)'; _nyxMigrationSqlRecord(sql, 'sequelize'); return sql; }},
|
||||
addColumn: async function(table, column){{ const sql = 'ALTER TABLE ' + String(table) + ' ADD COLUMN ' + String(column) + ' TEXT'; _nyxMigrationSqlRecord(sql, 'sequelize'); return sql; }},
|
||||
dropTable: async function(name){{ const sql = 'DROP TABLE ' + String(name); _nyxMigrationSqlRecord(sql, 'sequelize'); return sql; }},
|
||||
removeColumn: async function(table, column){{ const sql = 'ALTER TABLE ' + String(table) + ' DROP COLUMN ' + String(column); _nyxMigrationSqlRecord(sql, 'sequelize'); return sql; }},
|
||||
bulkInsert: async function(table){{ const sql = 'INSERT INTO ' + String(table) + ' VALUES (...)'; _nyxMigrationSqlRecord(sql, 'sequelize'); return sql; }},
|
||||
sequelize: {{ query: async function(sql){{ _nyxMigrationSqlRecord(sql, 'sequelize'); return sql; }} }},
|
||||
}};
|
||||
const _prisma = {{
|
||||
$executeRaw: async function(){{}},
|
||||
$executeRawUnsafe: async function(s){{ if (s) process.stdout.write('NYX_PRISMA_SQL: ' + s + '\n'); }},
|
||||
$queryRaw: async function(){{}},
|
||||
$queryRawUnsafe: async function(){{}},
|
||||
$executeRaw: async function(s){{ if (s) _nyxMigrationSqlRecord(s, 'prisma'); return s; }},
|
||||
$executeRawUnsafe: async function(s){{ if (s) {{ _nyxMigrationSqlRecord(s, 'prisma'); process.stdout.write('NYX_PRISMA_SQL: ' + s + '\n'); }} return s; }},
|
||||
$queryRaw: async function(s){{ if (s) _nyxMigrationSqlRecord(s, 'prisma'); return s; }},
|
||||
$queryRawUnsafe: async function(s){{ if (s) _nyxMigrationSqlRecord(s, 'prisma'); return s; }},
|
||||
}};
|
||||
global.__nyx_prisma = _prisma;
|
||||
(async () => {{
|
||||
try {{
|
||||
let _result;
|
||||
|
|
@ -1216,6 +1242,7 @@ const _prisma = {{
|
|||
_result = await Promise.resolve(_h());
|
||||
}}
|
||||
}}
|
||||
if (typeof _result === 'string') _nyxMigrationSqlRecord(_result, 'migration');
|
||||
if (_result != null) process.stdout.write(String(_result) + '\n');
|
||||
}} catch (e) {{
|
||||
process.stderr.write('NYX_EXCEPTION: ' + (e.constructor ? e.constructor.name : 'Error') + ': ' + e.message + '\n');
|
||||
|
|
|
|||
|
|
@ -3164,11 +3164,25 @@ fn emit_migration_harness(spec: &HarnessSpec, version: Option<&str>) -> HarnessS
|
|||
r#"{preamble}
|
||||
echo "__NYX_MIGRATION__: " . {version:?} . "\n";
|
||||
|
||||
function __nyx_migration_sqlish($value): bool {{
|
||||
$upper = strtoupper((string)$value);
|
||||
foreach (['SELECT', 'INSERT', 'UPDATE', 'DELETE', 'CREATE', 'ALTER', 'DROP'] as $kw) {{
|
||||
if (strpos($upper, $kw) !== false) return true;
|
||||
}}
|
||||
return false;
|
||||
}}
|
||||
|
||||
function __nyx_record_migration_result($value, string $driver): void {{
|
||||
if ($value === null || !__nyx_migration_sqlish($value)) return;
|
||||
__nyx_stub_sql_record((string)$value, ['driver' => $driver, 'source' => 'migration']);
|
||||
}}
|
||||
|
||||
if (class_exists({handler:?})) {{
|
||||
$inst = new {handler}();
|
||||
if (method_exists($inst, 'up')) {{
|
||||
try {{
|
||||
$result = $inst->up();
|
||||
__nyx_record_migration_result($result, 'laravel');
|
||||
if ($result !== null) echo (string)$result . "\n";
|
||||
}} catch (Throwable $e) {{
|
||||
fwrite(STDERR, 'NYX_EXCEPTION: ' . get_class($e) . ': ' . $e->getMessage() . "\n");
|
||||
|
|
@ -3180,6 +3194,7 @@ if (class_exists({handler:?})) {{
|
|||
}} elseif (function_exists({handler:?})) {{
|
||||
try {{
|
||||
$result = call_user_func({handler:?});
|
||||
__nyx_record_migration_result($result, 'php');
|
||||
if ($result !== null) echo (string)$result . "\n";
|
||||
}} catch (Throwable $e) {{
|
||||
fwrite(STDERR, 'NYX_EXCEPTION: ' . get_class($e) . ': ' . $e->getMessage() . "\n");
|
||||
|
|
|
|||
|
|
@ -968,7 +968,12 @@ def _nyx_sqs_dispatch(envelope):
|
|||
_loop.subscribe({queue:?}, _nyx_sqs_dispatch)
|
||||
print({publish_marker:?} + " " + {queue:?}, flush=True)
|
||||
_nyx_record_broker_publish("NYX_SQS_LOG", {queue:?}, payload)
|
||||
_loop.publish({queue:?}, payload)"#,
|
||||
_loop.publish({queue:?}, payload)
|
||||
for _env in _loop.receive_message({queue:?}, max_number=1):
|
||||
_nyx_record_broker_event("NYX_SQS_LOG", "deliver", {queue:?}, _env.get("Body", ""))
|
||||
_nyx_sqs_dispatch(_env)
|
||||
if _loop.delete_message({queue:?}, _env.get("ReceiptHandle", "")):
|
||||
_nyx_record_broker_event("NYX_SQS_LOG", "ack", {queue:?}, _env.get("ReceiptHandle", ""))"#,
|
||||
handler = handler,
|
||||
queue = queue,
|
||||
publish_marker = crate::dynamic::stubs::SQS_PUBLISH_MARKER,
|
||||
|
|
@ -1016,7 +1021,12 @@ def _nyx_kafka_dispatch(message):
|
|||
_loop.subscribe({queue:?}, _nyx_kafka_dispatch)
|
||||
print({publish_marker:?} + " " + {queue:?}, flush=True)
|
||||
_nyx_record_broker_publish("NYX_KAFKA_LOG", {queue:?}, payload)
|
||||
_loop.publish({queue:?}, payload)"#,
|
||||
_loop.publish({queue:?}, payload)
|
||||
for _record in _loop.poll({queue:?}, max_records=1):
|
||||
_nyx_record_broker_event("NYX_KAFKA_LOG", "deliver", {queue:?}, _record.value)
|
||||
_nyx_kafka_dispatch(_record.value)
|
||||
_loop.commit(_record)
|
||||
_nyx_record_broker_event("NYX_KAFKA_LOG", "ack", {queue:?}, str(_record.offset))"#,
|
||||
handler = handler,
|
||||
queue = queue,
|
||||
publish_marker = crate::dynamic::stubs::KAFKA_PUBLISH_MARKER,
|
||||
|
|
@ -1030,16 +1040,23 @@ _loop.publish({queue:?}, payload)"#,
|
|||
{pubsub_src}
|
||||
{rabbit_src}
|
||||
|
||||
def _nyx_record_broker_publish(env_name, destination, body):
|
||||
def _nyx_record_broker_event(env_name, action, destination, body):
|
||||
path = os.environ.get(env_name, "")
|
||||
if not path:
|
||||
return
|
||||
try:
|
||||
with open(path, "a", encoding="utf-8") as f:
|
||||
f.write(str(destination).replace("\t", " ") + "\t" + str(body) + "\n")
|
||||
f.write(
|
||||
str(action).replace("\t", " ") + "\t" +
|
||||
str(destination).replace("\t", " ") + "\t" +
|
||||
str(body) + "\n"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _nyx_record_broker_publish(env_name, destination, body):
|
||||
_nyx_record_broker_event(env_name, "publish", destination, body)
|
||||
|
||||
try:
|
||||
{register_and_publish}
|
||||
except SystemExit as _e:
|
||||
|
|
@ -1278,7 +1295,53 @@ _h = getattr(_entry_mod, {handler:?}, None)
|
|||
if _h is None:
|
||||
print("NYX_HANDLER_NOT_FOUND: " + {handler:?}, file=sys.stderr, flush=True)
|
||||
sys.exit(78)
|
||||
|
||||
def _nyx_migration_sql_record(sql, driver):
|
||||
text = str(sql)
|
||||
upper = text.upper()
|
||||
if not any(k in upper for k in ("SELECT", "INSERT", "UPDATE", "DELETE", "CREATE", "ALTER", "DROP")):
|
||||
return
|
||||
__nyx_stub_sql_record(text, driver=driver, source="migration")
|
||||
endpoint = os.environ.get("NYX_SQL_ENDPOINT", "")
|
||||
if endpoint:
|
||||
try:
|
||||
import sqlite3
|
||||
conn = sqlite3.connect(endpoint)
|
||||
try:
|
||||
conn.execute(text)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
class _NyxMigrationOpProxy:
|
||||
def __init__(self, inner=None):
|
||||
self._inner = inner
|
||||
def execute(self, sql, *args, **kwargs):
|
||||
_nyx_migration_sql_record(sql, "alembic")
|
||||
if self._inner is not None and self._inner is not self and hasattr(self._inner, "execute"):
|
||||
return self._inner.execute(sql, *args, **kwargs)
|
||||
return None
|
||||
|
||||
def _nyx_install_migration_sql_hooks():
|
||||
if hasattr(_entry_mod, "op"):
|
||||
try:
|
||||
_entry_mod.op = _NyxMigrationOpProxy(getattr(_entry_mod, "op"))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _nyx_record_migration_result(result):
|
||||
if result is None:
|
||||
return
|
||||
sql = getattr(result, "sql", None)
|
||||
if sql is not None:
|
||||
_nyx_migration_sql_record(sql, "django")
|
||||
elif isinstance(result, str):
|
||||
_nyx_migration_sql_record(result, "migration")
|
||||
|
||||
try:
|
||||
_nyx_install_migration_sql_hooks()
|
||||
# Migrations conventionally take no arguments; pass payload if the
|
||||
# function declares positional params (best-effort introspection).
|
||||
import inspect
|
||||
|
|
@ -1291,6 +1354,7 @@ try:
|
|||
_result = _h(payload)
|
||||
else:
|
||||
_result = _h()
|
||||
_nyx_record_migration_result(_result)
|
||||
if _result is not None:
|
||||
try:
|
||||
print(str(_result), flush=True)
|
||||
|
|
|
|||
|
|
@ -899,15 +899,34 @@ fn emit_migration_harness(spec: &HarnessSpec, version: Option<&str>) -> HarnessS
|
|||
r#"{preamble}
|
||||
puts "__NYX_MIGRATION__: " + {ver:?}
|
||||
|
||||
def __nyx_migration_sqlish?(value)
|
||||
text = value.to_s.upcase
|
||||
['SELECT', 'INSERT', 'UPDATE', 'DELETE', 'CREATE', 'ALTER', 'DROP'].any? {{ |k| text.include?(k) }}
|
||||
end
|
||||
|
||||
def __nyx_record_migration_result(value, driver)
|
||||
return if value.nil?
|
||||
return unless __nyx_migration_sqlish?(value)
|
||||
__nyx_stub_sql_record(value, driver: driver, source: 'migration')
|
||||
end
|
||||
|
||||
# ActiveRecord migrations expose `up` / `down` / `change` on a subclass.
|
||||
if Object.const_defined?({handler:?})
|
||||
cls = Object.const_get({handler:?})
|
||||
begin
|
||||
inst = cls.new
|
||||
if inst.respond_to?(:execute, true)
|
||||
original_execute = inst.method(:execute)
|
||||
inst.define_singleton_method(:execute) do |sql, *args, &blk|
|
||||
__nyx_record_migration_result(sql, 'active_record')
|
||||
original_execute.call(sql, *args, &blk)
|
||||
end
|
||||
end
|
||||
%i[up change down].each do |m|
|
||||
if inst.respond_to?(m)
|
||||
begin
|
||||
result = inst.send(m)
|
||||
__nyx_record_migration_result(result, 'active_record')
|
||||
print(result.to_s) if result
|
||||
rescue StandardError => e
|
||||
STDERR.puts("NYX_EXCEPTION: #{{e.class.name}}: #{{e.message}}")
|
||||
|
|
@ -923,6 +942,7 @@ end
|
|||
if respond_to?({handler:?}.to_sym, true)
|
||||
begin
|
||||
result = send({handler:?}.to_sym)
|
||||
__nyx_record_migration_result(result, 'ruby')
|
||||
print(result.to_s) if result
|
||||
rescue StandardError => e
|
||||
STDERR.puts("NYX_EXCEPTION: #{{e.class.name}}: #{{e.message}}")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue