diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 24102a23..785036c9 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -20,6 +20,18 @@ scrape_configs: - targets: - 'pulsar:8080' + - job_name: 'bookie' + scrape_interval: 5s + static_configs: + - targets: + - 'bookie:8000' + + - job_name: 'zookeeper' + scrape_interval: 5s + static_configs: + - targets: + - 'zookeeper:8000' + - job_name: 'pdf-decoder' scrape_interval: 5s static_configs: diff --git a/templates/components/pulsar.jsonnet b/templates/components/pulsar.jsonnet index 0342b4d5..58170500 100644 --- a/templates/components/pulsar.jsonnet +++ b/templates/components/pulsar.jsonnet @@ -2,31 +2,112 @@ local base = import "base/base.jsonnet"; local images = import "values/images.jsonnet"; local url = import "values/url.jsonnet"; +// This is a Pulsar configuration. Non-standalone mode so we deploy +// individual components: bookkeeper, broker and zookeeper. +// +// This also deploys the TrustGraph 'admin' container which initialises +// TrustGraph-specific namespaces etc. + { "pulsar" +: { create:: function(engine) -// local confVolume = engine.volume("pulsar-conf").with_size("2G"); - local dataVolume = engine.volume("pulsar-data").with_size("20G"); + // Zookeeper volume + local zkVolume = engine.volume("zookeeper").with_size("1G"); - local container = + // Zookeeper container + local zkContainer = + engine.container("zookeeper") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/zookeeper.conf && bin/generate-zookeeper-config.sh conf/zookeeper.conf && exec bin/pulsar zookeeper" + ]) + .with_limits("0.1", "400M") + .with_reservations("0.05", "400M") + .with_volume_mount(zkVolume, "/pulsar/data/zookeeper") + .with_environment({ + "metadataStoreUrl": "zk:zookeeper:2181", + "PULSAR_MEM": "-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m", + }) + .with_port(2181, 2181, "zookeeper") + .with_port(2888, 2888, "zookeeper2") + .with_port(3888, 3888, "zookeeper3"); + + // Pulsar cluster init container + local initContainer = + engine.container("pulsar-init") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "sleep 10 && bin/pulsar initialize-cluster-metadata --cluster cluster-a --zookeeper zookeeper:2181 --configuration-store zookeeper:2181 --web-service-url http://pulsar:8080 --broker-service-url pulsar://pulsar:6650", + ]) + .with_limits("1", "512M") + .with_reservations("0.05", "512M") + .with_environment({ + "PULSAR_MEM": "-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m", + }); + + + // Bookkeeper volume + local bookieVolume = engine.volume("bookie").with_size("20G"); + + // Bookkeeper container + local bookieContainer = + engine.container("bookie") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie" + // false ^ causes this to be a 'failure' exit. + ]) + .with_limits("1", "800M") + .with_reservations("0.1", "800M") + .with_user(0) + .with_volume_mount(bookieVolume, "/pulsar/data/bookkeeper") + .with_environment({ + "clusterName": "cluster-a", + "zkServers": "zookeeper:2181", + "bookieId": "bookie", + "metadataStoreUri": "metadata-store:zk:zookeeper:2181", + "advertisedAddress": "bookie", + "BOOKIE_MEM": "-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m", + }) + .with_port(3181, 3181, "bookie"); + + // Pulsar broker, stateless (uses ZK and Bookkeeper for state) + local brokerContainer = engine.container("pulsar") .with_image(images.pulsar) - .with_command(["bin/pulsar", "standalone"]) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" + ]) + .with_limits("1", "800M") + .with_reservations("0.1", "800M") .with_environment({ - "PULSAR_MEM": "-Xms600M -Xmx600M" + "metadataStoreUrl": "zk:zookeeper:2181", + "zookeeperServers": "zookeeper:2181", + "clusterName": "cluster-a", + "managedLedgerDefaultEnsembleSize": "1", + "managedLedgerDefaultWriteQuorum": "1", + "managedLedgerDefaultAckQuorum": "1", + "advertisedAddress": "pulsar", + "advertisedListeners": "external:pulsar://pulsar:6650", + "PULSAR_MEM": "-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m", }) - .with_limits("2.0", "1500M") - .with_reservations("1.0", "1500M") -// .with_volume_mount(confVolume, "/pulsar/conf") - .with_volume_mount(dataVolume, "/pulsar/data") - .with_port(6650, 6650, "bookie") - .with_port(8080, 8080, "http"); + .with_port(6650, 6650, "pulsar") + .with_port(8080, 8080, "admin"); + // Trustgraph Pulsar initialisation local adminContainer = - engine.container("init-pulsar") + engine.container("init-trustgraph") .with_image(images.trustgraph) .with_command([ "tg-init-pulsar", @@ -36,10 +117,32 @@ local url = import "values/url.jsonnet"; .with_limits("1", "128M") .with_reservations("0.1", "128M"); - local containerSet = engine.containers( - "pulsar", + // Container sets + local zkContainerSet = engine.containers( + "zookeeper", [ - container + zkContainer, + ] + ); + + local initContainerSet = engine.containers( + "init-pulsar", + [ + initContainer, + ] + ); + + local bookieContainerSet = engine.containers( + "bookie", + [ + bookieContainer, + ] + ); + + local brokerContainerSet = engine.containers( + "broker", + [ + brokerContainer, ] ); @@ -50,17 +153,35 @@ local url = import "values/url.jsonnet"; ] ); - local service = - engine.service(containerSet) - .with_port(6650, 6650, "bookie") - .with_port(8080, 8080, "http"); + // Zookeeper service + local zkService = + engine.service(zkContainerSet) + .with_port(2181, 2181, "zookeeper") + .with_port(2888, 2888, "zookeeper2") + .with_port(3888, 3888, "zookeeper3"); + + // Bookkeeper service + local bookieService = + engine.service(bookieContainerSet) + .with_port(3181, 3181, "bookie"); + + // Pulsar broker service + local brokerService = + engine.service(brokerContainerSet) + .with_port(6650, 6650, "pulsar") + .with_port(8080, 8080, "admin"); engine.resources([ -// confVolume, - dataVolume, - containerSet, + zkVolume, + bookieVolume, + zkContainerSet, + initContainerSet, + bookieContainerSet, + brokerContainerSet, adminContainerSet, - service, + zkService, + bookieService, + brokerService, ]) } diff --git a/templates/engine/docker-compose.jsonnet b/templates/engine/docker-compose.jsonnet index c37f1df0..0be3c3e3 100644 --- a/templates/engine/docker-compose.jsonnet +++ b/templates/engine/docker-compose.jsonnet @@ -22,6 +22,8 @@ with_image:: function(x) self + { image: x }, + with_user:: function(x) self + { user: x }, + with_command:: function(x) self + { command: x }, with_environment:: function(x) self + { @@ -75,6 +77,10 @@ { command: container.command } else {}) + + (if std.objectHas(container, "user") then + { user: container.user } + else {}) + + (if ! std.isEmpty(container.environment) then { environment: container.environment } else {}) +