mirror of
https://github.com/elicpeter/nyx.git
synced 2026-06-21 20:18:06 +02:00
refactor(dynamic): add SQS loopback HTTP emulator with real SDK compatibility, extend stub event recording and endpoint rewriting logic across Java and Python
This commit is contained in:
elipeter
parent
0903231189
commit
433036aead
6 changed files with 765 additions and 39 deletions
|
|
@ -3803,25 +3803,27 @@ fn emit_message_handler_harness(
|
|||
JavaBroker::Sqs => (
|
||||
crate::dynamic::stubs::SQS_PUBLISH_MARKER,
|
||||
format!(
|
||||
r#" NyxSqsLoopback brokerRef = new NyxSqsLoopback();
|
||||
System.out.println({publish_marker:?} + " " + {queue:?});
|
||||
nyxRecordBrokerPublish("NYX_SQS_LOG", {queue:?}, payload);
|
||||
brokerRef.publish({queue:?}, payload);
|
||||
for (java.util.Map<String, String> env : brokerRef.receiveMessage({queue:?}, 1)) {{
|
||||
nyxRecordBrokerEvent("NYX_SQS_LOG", "deliver", {queue:?}, env.getOrDefault("Body", ""));
|
||||
System.out.println("__NYX_SINK_HIT__");
|
||||
boolean success = false;
|
||||
try {{
|
||||
java.lang.reflect.Method m = entryInst.getClass().getDeclaredMethod({handler:?}, java.util.Map.class);
|
||||
m.setAccessible(true);
|
||||
m.invoke(entryInst, env);
|
||||
success = true;
|
||||
}} catch (Exception e) {{
|
||||
Throwable c = (e instanceof java.lang.reflect.InvocationTargetException && e.getCause() != null) ? e.getCause() : e;
|
||||
System.err.println("NYX_EXCEPTION: " + c.getClass().getName() + ": " + c.getMessage());
|
||||
}}
|
||||
if (success && brokerRef.deleteMessage({queue:?}, env.getOrDefault("ReceiptHandle", ""))) {{
|
||||
nyxRecordBrokerEvent("NYX_SQS_LOG", "ack", {queue:?}, env.getOrDefault("ReceiptHandle", ""));
|
||||
r#" if (!nyxTryRealSqs({queue:?}, payload, entryInst, {handler:?})) {{
|
||||
NyxSqsLoopback brokerRef = new NyxSqsLoopback();
|
||||
System.out.println({publish_marker:?} + " " + {queue:?});
|
||||
nyxRecordBrokerPublish("NYX_SQS_LOG", {queue:?}, payload);
|
||||
brokerRef.publish({queue:?}, payload);
|
||||
for (java.util.Map<String, String> env : brokerRef.receiveMessage({queue:?}, 1)) {{
|
||||
nyxRecordBrokerEvent("NYX_SQS_LOG", "deliver", {queue:?}, env.getOrDefault("Body", ""));
|
||||
System.out.println("__NYX_SINK_HIT__");
|
||||
boolean success = false;
|
||||
try {{
|
||||
java.lang.reflect.Method m = entryInst.getClass().getDeclaredMethod({handler:?}, java.util.Map.class);
|
||||
m.setAccessible(true);
|
||||
m.invoke(entryInst, env);
|
||||
success = true;
|
||||
}} catch (Exception e) {{
|
||||
Throwable c = (e instanceof java.lang.reflect.InvocationTargetException && e.getCause() != null) ? e.getCause() : e;
|
||||
System.err.println("NYX_EXCEPTION: " + c.getClass().getName() + ": " + c.getMessage());
|
||||
}}
|
||||
if (success && brokerRef.deleteMessage({queue:?}, env.getOrDefault("ReceiptHandle", ""))) {{
|
||||
nyxRecordBrokerEvent("NYX_SQS_LOG", "ack", {queue:?}, env.getOrDefault("ReceiptHandle", ""));
|
||||
}}
|
||||
}}
|
||||
}}"#,
|
||||
handler = handler,
|
||||
|
|
@ -3926,6 +3928,97 @@ public class NyxHarness {{
|
|||
}}
|
||||
}}
|
||||
|
||||
static boolean nyxTryRealSqs(String queue, String payload, Object entryInst, String handler) {{
|
||||
String endpoint = System.getenv("NYX_SQS_ENDPOINT");
|
||||
if (endpoint == null || !(endpoint.startsWith("http://") || endpoint.startsWith("https://"))) {{
|
||||
return false;
|
||||
}}
|
||||
Object client = null;
|
||||
try {{
|
||||
Class<?> sqsClientClass = Class.forName("software.amazon.awssdk.services.sqs.SqsClient");
|
||||
Object builder = sqsClientClass.getMethod("builder").invoke(null);
|
||||
Class<?> regionClass = Class.forName("software.amazon.awssdk.regions.Region");
|
||||
Object region = regionClass.getMethod("of", String.class).invoke(null, "us-east-1");
|
||||
builder.getClass().getMethod("endpointOverride", java.net.URI.class)
|
||||
.invoke(builder, java.net.URI.create(endpoint));
|
||||
builder.getClass().getMethod("region", regionClass).invoke(builder, region);
|
||||
|
||||
Class<?> basicCredentialsClass = Class.forName("software.amazon.awssdk.auth.credentials.AwsBasicCredentials");
|
||||
Class<?> credentialsClass = Class.forName("software.amazon.awssdk.auth.credentials.AwsCredentials");
|
||||
Class<?> providerClass = Class.forName("software.amazon.awssdk.auth.credentials.StaticCredentialsProvider");
|
||||
Class<?> providerInterface = Class.forName("software.amazon.awssdk.auth.credentials.AwsCredentialsProvider");
|
||||
Object credentials = basicCredentialsClass.getMethod("create", String.class, String.class)
|
||||
.invoke(null, "nyx", "nyx");
|
||||
Object provider = providerClass.getMethod("create", credentialsClass).invoke(null, credentials);
|
||||
builder.getClass().getMethod("credentialsProvider", providerInterface).invoke(builder, provider);
|
||||
client = builder.getClass().getMethod("build").invoke(builder);
|
||||
|
||||
String queueUrl = endpoint.replaceAll("/+$", "") + "/" + queue.replaceAll("^/+", "");
|
||||
System.out.println({sqs_publish_marker:?} + " " + queue);
|
||||
|
||||
Class<?> sendReqClass = Class.forName("software.amazon.awssdk.services.sqs.model.SendMessageRequest");
|
||||
Object sendBuilder = sendReqClass.getMethod("builder").invoke(null);
|
||||
sendBuilder.getClass().getMethod("queueUrl", String.class).invoke(sendBuilder, queueUrl);
|
||||
sendBuilder.getClass().getMethod("messageBody", String.class).invoke(sendBuilder, payload);
|
||||
Object sendReq = sendBuilder.getClass().getMethod("build").invoke(sendBuilder);
|
||||
sqsClientClass.getMethod("sendMessage", sendReqClass).invoke(client, sendReq);
|
||||
|
||||
Class<?> receiveReqClass = Class.forName("software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest");
|
||||
Object receiveBuilder = receiveReqClass.getMethod("builder").invoke(null);
|
||||
receiveBuilder.getClass().getMethod("queueUrl", String.class).invoke(receiveBuilder, queueUrl);
|
||||
receiveBuilder.getClass().getMethod("maxNumberOfMessages", Integer.class).invoke(receiveBuilder, Integer.valueOf(1));
|
||||
receiveBuilder.getClass().getMethod("waitTimeSeconds", Integer.class).invoke(receiveBuilder, Integer.valueOf(0));
|
||||
Object receiveReq = receiveBuilder.getClass().getMethod("build").invoke(receiveBuilder);
|
||||
Object receiveResp = sqsClientClass.getMethod("receiveMessage", receiveReqClass).invoke(client, receiveReq);
|
||||
java.util.List<?> messages = (java.util.List<?>) receiveResp.getClass().getMethod("messages").invoke(receiveResp);
|
||||
if (messages == null || messages.isEmpty()) {{
|
||||
return false;
|
||||
}}
|
||||
|
||||
for (Object msg : messages) {{
|
||||
String body = String.valueOf(msg.getClass().getMethod("body").invoke(msg));
|
||||
String receipt = String.valueOf(msg.getClass().getMethod("receiptHandle").invoke(msg));
|
||||
String messageId = String.valueOf(msg.getClass().getMethod("messageId").invoke(msg));
|
||||
java.util.Map<String, String> env = new java.util.HashMap<>();
|
||||
env.put("Body", body);
|
||||
env.put("ReceiptHandle", receipt);
|
||||
env.put("MessageId", messageId);
|
||||
System.out.println("__NYX_SINK_HIT__");
|
||||
boolean success = false;
|
||||
try {{
|
||||
java.lang.reflect.Method m = entryInst.getClass().getDeclaredMethod(handler, java.util.Map.class);
|
||||
m.setAccessible(true);
|
||||
m.invoke(entryInst, env);
|
||||
success = true;
|
||||
}} catch (Exception e) {{
|
||||
Throwable c = (e instanceof java.lang.reflect.InvocationTargetException && e.getCause() != null) ? e.getCause() : e;
|
||||
System.err.println("NYX_EXCEPTION: " + c.getClass().getName() + ": " + c.getMessage());
|
||||
}}
|
||||
if (success && receipt != null && !receipt.isEmpty()) {{
|
||||
Class<?> deleteReqClass = Class.forName("software.amazon.awssdk.services.sqs.model.DeleteMessageRequest");
|
||||
Object deleteBuilder = deleteReqClass.getMethod("builder").invoke(null);
|
||||
deleteBuilder.getClass().getMethod("queueUrl", String.class).invoke(deleteBuilder, queueUrl);
|
||||
deleteBuilder.getClass().getMethod("receiptHandle", String.class).invoke(deleteBuilder, receipt);
|
||||
Object deleteReq = deleteBuilder.getClass().getMethod("build").invoke(deleteBuilder);
|
||||
sqsClientClass.getMethod("deleteMessage", deleteReqClass).invoke(client, deleteReq);
|
||||
}}
|
||||
}}
|
||||
return true;
|
||||
}} catch (ClassNotFoundException missingSdk) {{
|
||||
return false;
|
||||
}} catch (Throwable e) {{
|
||||
System.err.println("NYX_REAL_SQS_FALLBACK: " + e.getClass().getName() + ": " + e.getMessage());
|
||||
return false;
|
||||
}} finally {{
|
||||
if (client instanceof AutoCloseable) {{
|
||||
try {{
|
||||
((AutoCloseable) client).close();
|
||||
}} catch (Exception ignored) {{
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
|
||||
static String nyxPayload() {{
|
||||
String v = System.getenv("NYX_PAYLOAD");
|
||||
if (v != null && !v.isEmpty()) return v;
|
||||
|
|
@ -3959,6 +4052,7 @@ public class NyxHarness {{
|
|||
"#,
|
||||
entry_class = entry_class,
|
||||
dispatch_block = dispatch_block,
|
||||
sqs_publish_marker = crate::dynamic::stubs::SQS_PUBLISH_MARKER,
|
||||
);
|
||||
HarnessSource {
|
||||
source,
|
||||
|
|
@ -3966,7 +4060,7 @@ public class NyxHarness {{
|
|||
command: vec![
|
||||
"java".to_owned(),
|
||||
"-cp".to_owned(),
|
||||
".".to_owned(),
|
||||
".:lib/*".to_owned(),
|
||||
"NyxHarness".to_owned(),
|
||||
],
|
||||
extra_files: {
|
||||
|
|
|
|||
|
|
@ -945,6 +945,53 @@ function _nyxRecordBrokerPublish(envName, destination, body) {{
|
|||
_nyxRecordBrokerEvent(envName, 'publish', destination, body);
|
||||
}}
|
||||
|
||||
async function _nyxTryRealSqs(queue, body) {{
|
||||
const endpoint = process.env.NYX_SQS_ENDPOINT || '';
|
||||
if (!/^https?:\/\//.test(endpoint)) return false;
|
||||
let sqs;
|
||||
try {{
|
||||
sqs = require('@aws-sdk/client-sqs');
|
||||
}} catch (_) {{
|
||||
return false;
|
||||
}}
|
||||
try {{
|
||||
const client = new sqs.SQSClient({{
|
||||
endpoint,
|
||||
region: 'us-east-1',
|
||||
credentials: {{ accessKeyId: 'nyx', secretAccessKey: 'nyx' }},
|
||||
maxAttempts: 1,
|
||||
}});
|
||||
const queueUrl = endpoint.replace(/\/$/, '') + '/' + String(queue).replace(/^\//, '');
|
||||
process.stdout.write({publish_marker:?} + ' ' + queue + '\n');
|
||||
await client.send(new sqs.SendMessageCommand({{
|
||||
QueueUrl: queueUrl,
|
||||
MessageBody: String(body),
|
||||
}}));
|
||||
const response = await client.send(new sqs.ReceiveMessageCommand({{
|
||||
QueueUrl: queueUrl,
|
||||
MaxNumberOfMessages: 1,
|
||||
WaitTimeSeconds: 0,
|
||||
AttributeNames: ['ApproximateReceiveCount'],
|
||||
}}));
|
||||
const messages = response.Messages || [];
|
||||
if (messages.length === 0) return false;
|
||||
for (const envelope of messages) {{
|
||||
const ok = await _nyxDispatchEnvelope(envelope);
|
||||
if (ok && envelope.ReceiptHandle) {{
|
||||
await client.send(new sqs.DeleteMessageCommand({{
|
||||
QueueUrl: queueUrl,
|
||||
ReceiptHandle: envelope.ReceiptHandle,
|
||||
}}));
|
||||
}}
|
||||
}}
|
||||
if (typeof client.destroy === 'function') client.destroy();
|
||||
return true;
|
||||
}} catch (e) {{
|
||||
process.stderr.write('NYX_REAL_SQS_FALLBACK: ' + (e && e.message ? e.message : String(e)) + '\n');
|
||||
return false;
|
||||
}}
|
||||
}}
|
||||
|
||||
async function _nyxDispatchEnvelope(envelope) {{
|
||||
try {{
|
||||
// Sink-reachability sentinel — runner's `vuln_fired && sink_hit`
|
||||
|
|
@ -959,6 +1006,7 @@ async function _nyxDispatchEnvelope(envelope) {{
|
|||
}}
|
||||
|
||||
(async () => {{
|
||||
if (await _nyxTryRealSqs({queue:?}, payload)) return;
|
||||
process.stdout.write({publish_marker:?} + ' ' + {queue:?} + '\n');
|
||||
_nyxRecordBrokerPublish('NYX_SQS_LOG', {queue:?}, payload);
|
||||
_broker.publish({queue:?}, payload);
|
||||
|
|
|
|||
|
|
@ -958,22 +958,23 @@ fn emit_message_handler(spec: &HarnessSpec, queue: &str) -> HarnessSource {
|
|||
|
||||
let register_and_publish = match broker {
|
||||
PythonBroker::Sqs => format!(
|
||||
r#"_loop = NyxSqsLoopback()
|
||||
def _nyx_sqs_dispatch(envelope):
|
||||
_h = getattr(_entry_mod, {handler:?}, None)
|
||||
if _h is None:
|
||||
print("NYX_HANDLER_NOT_FOUND: " + {handler:?}, file=sys.stderr, flush=True)
|
||||
sys.exit(78)
|
||||
_h(envelope)
|
||||
_loop.subscribe({queue:?}, _nyx_sqs_dispatch)
|
||||
print({publish_marker:?} + " " + {queue:?}, flush=True)
|
||||
_nyx_record_broker_publish("NYX_SQS_LOG", {queue:?}, payload)
|
||||
_loop.publish({queue:?}, payload)
|
||||
for _env in _loop.receive_message({queue:?}, max_number=1):
|
||||
_nyx_record_broker_event("NYX_SQS_LOG", "deliver", {queue:?}, _env.get("Body", ""))
|
||||
_nyx_sqs_dispatch(_env)
|
||||
if _loop.delete_message({queue:?}, _env.get("ReceiptHandle", "")):
|
||||
_nyx_record_broker_event("NYX_SQS_LOG", "ack", {queue:?}, _env.get("ReceiptHandle", ""))"#,
|
||||
r#"if not _nyx_try_real_sqs({queue:?}, payload, {handler:?}):
|
||||
_loop = NyxSqsLoopback()
|
||||
def _nyx_sqs_dispatch(envelope):
|
||||
_h = getattr(_entry_mod, {handler:?}, None)
|
||||
if _h is None:
|
||||
print("NYX_HANDLER_NOT_FOUND: " + {handler:?}, file=sys.stderr, flush=True)
|
||||
sys.exit(78)
|
||||
_h(envelope)
|
||||
_loop.subscribe({queue:?}, _nyx_sqs_dispatch)
|
||||
print({publish_marker:?} + " " + {queue:?}, flush=True)
|
||||
_nyx_record_broker_publish("NYX_SQS_LOG", {queue:?}, payload)
|
||||
_loop.publish({queue:?}, payload)
|
||||
for _env in _loop.receive_message({queue:?}, max_number=1):
|
||||
_nyx_record_broker_event("NYX_SQS_LOG", "deliver", {queue:?}, _env.get("Body", ""))
|
||||
_nyx_sqs_dispatch(_env)
|
||||
if _loop.delete_message({queue:?}, _env.get("ReceiptHandle", "")):
|
||||
_nyx_record_broker_event("NYX_SQS_LOG", "ack", {queue:?}, _env.get("ReceiptHandle", ""))"#,
|
||||
handler = handler,
|
||||
queue = queue,
|
||||
publish_marker = crate::dynamic::stubs::SQS_PUBLISH_MARKER,
|
||||
|
|
@ -1063,6 +1064,61 @@ def _nyx_record_broker_event(env_name, action, destination, body):
|
|||
def _nyx_record_broker_publish(env_name, destination, body):
|
||||
_nyx_record_broker_event(env_name, "publish", destination, body)
|
||||
|
||||
def _nyx_try_real_sqs(queue, body, handler_name):
|
||||
endpoint = os.environ.get("NYX_SQS_ENDPOINT", "")
|
||||
if not (endpoint.startswith("http://") or endpoint.startswith("https://")):
|
||||
return False
|
||||
try:
|
||||
import boto3
|
||||
try:
|
||||
from botocore.config import Config
|
||||
_cfg = Config(
|
||||
retries={{"max_attempts": 0}},
|
||||
connect_timeout=1,
|
||||
read_timeout=2,
|
||||
)
|
||||
except Exception:
|
||||
_cfg = None
|
||||
except Exception:
|
||||
return False
|
||||
_h = getattr(_entry_mod, handler_name, None)
|
||||
if _h is None:
|
||||
print("NYX_HANDLER_NOT_FOUND: " + handler_name, file=sys.stderr, flush=True)
|
||||
sys.exit(78)
|
||||
try:
|
||||
_kwargs = {{
|
||||
"endpoint_url": endpoint,
|
||||
"region_name": "us-east-1",
|
||||
"aws_access_key_id": "nyx",
|
||||
"aws_secret_access_key": "nyx",
|
||||
}}
|
||||
if _cfg is not None:
|
||||
_kwargs["config"] = _cfg
|
||||
_client = boto3.client("sqs", **_kwargs)
|
||||
_queue_url = endpoint.rstrip("/") + "/" + str(queue).strip("/")
|
||||
print({sqs_publish_marker:?} + " " + str(queue), flush=True)
|
||||
_client.send_message(QueueUrl=_queue_url, MessageBody=str(body))
|
||||
_resp = _client.receive_message(
|
||||
QueueUrl=_queue_url,
|
||||
MaxNumberOfMessages=1,
|
||||
WaitTimeSeconds=0,
|
||||
AttributeNames=["ApproximateReceiveCount"],
|
||||
)
|
||||
_messages = _resp.get("Messages", [])
|
||||
if not _messages:
|
||||
return False
|
||||
for _msg in _messages:
|
||||
_h(_msg)
|
||||
_receipt = _msg.get("ReceiptHandle", "")
|
||||
if _receipt:
|
||||
_client.delete_message(QueueUrl=_queue_url, ReceiptHandle=_receipt)
|
||||
return True
|
||||
except SystemExit:
|
||||
raise
|
||||
except Exception as _e:
|
||||
print(f"NYX_REAL_SQS_FALLBACK: {{type(_e).__name__}}: {{_e}}", file=sys.stderr, flush=True)
|
||||
return False
|
||||
|
||||
try:
|
||||
{register_and_publish}
|
||||
except SystemExit as _e:
|
||||
|
|
@ -1075,6 +1131,7 @@ except Exception as _e:
|
|||
pubsub_src = pubsub_src,
|
||||
rabbit_src = rabbit_src,
|
||||
register_and_publish = indent_lines(®ister_and_publish, " "),
|
||||
sqs_publish_marker = crate::dynamic::stubs::SQS_PUBLISH_MARKER,
|
||||
);
|
||||
HarnessSource {
|
||||
source: format!("{preamble}\n{body}\n{postamble}"),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue