From 53f76b6333ac95b2f2fc49921910e82ee985d57a Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Tue, 21 Apr 2026 11:46:54 +0200 Subject: [PATCH 1/3] MINIFICPP-2775 Allow manifest generation for comparison between releases --- .../github_actions_cache_cleanup.py | 0 .../github_actions_cache_cleanup_tests.py | 0 .../github_scripts}/requirements.txt | 0 .../ubuntu_22_04_clang_arm_manifest.json | 11731 ++++++++++++++++ .github/workflows/ci.yml | 5 + .github/workflows/clear-actions-cache.yml | 4 +- cmake/ArgParse.cmake | 4 +- minifi_main/AgentDocs.cpp | 53 +- minifi_main/AgentDocs.h | 1 + minifi_main/MiNiFiMain.cpp | 23 +- 10 files changed, 11803 insertions(+), 18 deletions(-) rename {github_scripts => .github/github_scripts}/github_actions_cache_cleanup.py (100%) rename {github_scripts => .github/github_scripts}/github_actions_cache_cleanup_tests.py (100%) rename {github_scripts => .github/github_scripts}/requirements.txt (100%) create mode 100644 .github/references/ubuntu_22_04_clang_arm_manifest.json diff --git a/github_scripts/github_actions_cache_cleanup.py b/.github/github_scripts/github_actions_cache_cleanup.py similarity index 100% rename from github_scripts/github_actions_cache_cleanup.py rename to .github/github_scripts/github_actions_cache_cleanup.py diff --git a/github_scripts/github_actions_cache_cleanup_tests.py b/.github/github_scripts/github_actions_cache_cleanup_tests.py similarity index 100% rename from github_scripts/github_actions_cache_cleanup_tests.py rename to .github/github_scripts/github_actions_cache_cleanup_tests.py diff --git a/github_scripts/requirements.txt b/.github/github_scripts/requirements.txt similarity index 100% rename from github_scripts/requirements.txt rename to .github/github_scripts/requirements.txt diff --git a/.github/references/ubuntu_22_04_clang_arm_manifest.json b/.github/references/ubuntu_22_04_clang_arm_manifest.json new file mode 100644 index 0000000000..f9c6249137 --- /dev/null +++ b/.github/references/ubuntu_22_04_clang_arm_manifest.json @@ -0,0 +1,11731 @@ +{ + "componentManifest": { + "processors": [ + { + "propertyDescriptors": { + "Hostname Attribute": { + "name": "Hostname Attribute", + "description": "Flowfile attribute used to record the agent's hostname", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "source.hostname" + }, + "IP Attribute": { + "name": "IP Attribute", + "description": "Flowfile attribute used to record the agent's IP addresses in a comma separated list", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "source.ipv4" + }, + "Network Interface Filter": { + "name": "Network Interface Filter", + "description": "A regular expression to filter ip addresses based on the name of the network interface", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Refresh Policy": { + "name": "Refresh Policy", + "description": "When to recalculate the host info", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "On schedule", + "allowableValues": [ + { + "value": "On schedule", + "displayName": "On schedule" + }, + { + "value": "On every trigger", + "displayName": "On every trigger" + } + ] + } + }, + "inputRequirement": "INPUT_ALLOWED", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "success", + "description": "success operational on the flow record" + } + ], + "typeDescription": "Appends host information such as IP address and hostname as an attribute to incoming flowfiles.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "org.apache.nifi.minifi.processors.AppendHostInfo" + }, + { + "propertyDescriptors": { + "Template": { + "name": "Template", + "description": "Path to the input mustache template file", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + } + }, + "inputRequirement": "INPUT_ALLOWED", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "success", + "description": "success operational on the flow record" + } + ], + "typeDescription": "Applies the mustache template specified by the \"Template\" property and writes the output to the flow file content. FlowFile attributes are used as template parameters.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "org.apache.nifi.minifi.processors.ApplyTemplate" + }, + { + "propertyDescriptors": { + "Attribute name prefix": { + "name": "Attribute name prefix", + "description": "The prefix to add to the generated attribute names. For example, if this is set to 'rolling.window.', then the full attribute names will be 'rolling.window.value', 'rolling.window.count', etc.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "rolling.window." + }, + "Time window": { + "name": "Time window", + "description": "The amount of time for a rolling window. The format of the value is expected to be a count followed by a time unit. For example 5 millis, 10 secs, 1 min, 3 hours, 2 days, etc.", + "validator": "TIME_PERIOD_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Value to track": { + "name": "Value to track", + "description": "The expression on which to evaluate each FlowFile. The result of the expression will be added to the rolling window value.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES" + }, + "Window length": { + "name": "Window length", + "description": "The window length in number of values. Takes precedence over 'Time window'. If set to zero, the 'Time window' property is used instead.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0" + } + }, + "inputRequirement": "INPUT_REQUIRED", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "failure", + "description": "When a FlowFile fails, it is routed here." + }, + { + "name": "success", + "description": "All FlowFiles that are successfully processed are routed to this relationship." + } + ], + "typeDescription": "Track a Rolling Window based on evaluating an Expression Language expression on each FlowFile. Each FlowFile will be emitted with the count of FlowFiles and total aggregate value of values processed in the current window.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "org.apache.nifi.minifi.processors.AttributeRollingWindow" + }, + { + "propertyDescriptors": { + "Attributes List": { + "name": "Attributes List", + "description": "Comma separated list of attributes to be included in the resulting JSON. If this value is left empty then all existing Attributes will be included. This list of attributes is case sensitive. If an attribute specified in the list is not found it will be be emitted to the resulting JSON with an empty string or NULL value.", + "validator": "NON_BLANK_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Attributes Regular Expression": { + "name": "Attributes Regular Expression", + "description": "Regular expression that will be evaluated against the flow file attributes to select the matching attributes. Both the matching attributes and the selected attributes from the Attributes List property will be written in the resulting JSON.", + "validator": "NON_BLANK_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Destination": { + "name": "Destination", + "description": "Control if JSON value is written as a new flowfile attribute 'JSONAttributes' or written in the flowfile content. Writing to flowfile content will overwrite any existing flowfile content.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "flowfile-attribute", + "allowableValues": [ + { + "value": "flowfile-attribute", + "displayName": "flowfile-attribute" + }, + { + "value": "flowfile-content", + "displayName": "flowfile-content" + } + ] + }, + "Include Core Attributes": { + "name": "Include Core Attributes", + "description": "Determines if the FlowFile core attributes which are contained in every FlowFile should be included in the final JSON value generated.", + "validator": "BOOLEAN_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "true" + }, + "Null Value": { + "name": "Null Value", + "description": "If true a non existing selected attribute will be NULL in the resulting JSON. If false an empty string will be placed in the JSON.", + "validator": "BOOLEAN_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "false" + } + }, + "inputRequirement": "INPUT_REQUIRED", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "success", + "description": "All FlowFiles received are routed to success" + } + ], + "typeDescription": "Generates a JSON representation of the input FlowFile Attributes. The resulting JSON can be written to either a new Attribute 'JSONAttributes' or written to the FlowFile as content.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "org.apache.nifi.minifi.processors.AttributesToJSON" + }, + { + "propertyDescriptors": { + "Image Encoding": { + "name": "Image Encoding", + "description": "The encoding that should be applied the the frame images captured from the RTSP stream", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": ".jpg" + }, + "RTSP Hostname": { + "name": "RTSP Hostname", + "description": "Hostname of the RTSP stream we are trying to connect to", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "RTSP Password": { + "name": "RTSP Password", + "description": "Password used to connect to the RTSP stream", + "validator": "VALID", + "required": "false", + "sensitive": "true", + "expressionLanguageScope": "NONE" + }, + "RTSP Port": { + "name": "RTSP Port", + "description": "Port that should be connected to to receive RTSP Frames", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "RTSP URI": { + "name": "RTSP URI", + "description": "URI that should be appended to the RTSP stream hostname", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "RTSP Username": { + "name": "RTSP Username", + "description": "The username for connecting to the RTSP stream", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + } + }, + "inputRequirement": "INPUT_ALLOWED", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "failure", + "description": "Failures to capture RTSP frame" + }, + { + "name": "success", + "description": "Successful capture of RTSP frame" + } + ], + "typeDescription": "Captures a frame from the RTSP stream at specified intervals.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "org.apache.nifi.minifi.processors.CaptureRTSPFrame" + }, + { + "propertyDescriptors": { + "Kubernetes Controller Service": { + "typeProvidedByValue": { + "type": "org.apache.nifi.minifi.controllers.KubernetesControllerService", + "group": "org.apache.nifi.minifi", + "artifact": "minifi-kubernetes-extensions" + }, + "name": "Kubernetes Controller Service", + "description": "Controller service which provides Kubernetes functionality", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + } + }, + "inputRequirement": "INPUT_FORBIDDEN", + "isSingleThreaded": "true", + "supportedRelationships": [ + { + "name": "success", + "description": "All flow files produced are routed to Success." + } + ], + "typeDescription": "A processor which collects pod metrics when MiNiFi is run inside Kubernetes.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "org.apache.nifi.minifi.processors.CollectKubernetesPodMetrics" + }, + { + "propertyDescriptors": { + "Batch Size": { + "name": "Batch Size", + "description": "Maximum number of FlowFiles processed in a single session", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "1" + }, + "Compression Format": { + "name": "Compression Format", + "description": "The compression format to use.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "use mime.type attribute", + "allowableValues": [ + { + "value": "gzip", + "displayName": "gzip" + }, + { + "value": "lzma", + "displayName": "lzma" + }, + { + "value": "xz-lzma2", + "displayName": "xz-lzma2" + }, + { + "value": "bzip2", + "displayName": "bzip2" + }, + { + "value": "use mime.type attribute", + "displayName": "use mime.type attribute" + } + ] + }, + "Compression Level": { + "name": "Compression Level", + "description": "The compression level to use; this is valid only when using GZIP compression.", + "validator": "INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "1" + }, + "Encapsulate in TAR": { + "name": "Encapsulate in TAR", + "description": "If true, on compression the FlowFile is added to a TAR archive and then compressed, and on decompression a compressed, TAR-encapsulated FlowFile is expected.\nIf false, on compression the content of the FlowFile simply gets compressed, and on decompression a simple compressed content is expected.\ntrue is the behaviour compatible with older MiNiFi C++ versions, false is the behaviour compatible with NiFi.", + "validator": "BOOLEAN_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "true" + }, + "Mode": { + "name": "Mode", + "description": "Indicates whether the processor should compress content or decompress content.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "compress", + "allowableValues": [ + { + "value": "compress", + "displayName": "compress" + }, + { + "value": "decompress", + "displayName": "decompress" + } + ] + }, + "Update Filename": { + "name": "Update Filename", + "description": "Determines if filename extension need to be updated", + "validator": "BOOLEAN_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "false" + } + }, + "inputRequirement": "INPUT_REQUIRED", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "failure", + "description": "FlowFiles will be transferred to the failure relationship if they fail to compress/decompress" + }, + { + "name": "success", + "description": "FlowFiles will be transferred to the success relationship after successfully being compressed or decompressed" + } + ], + "typeDescription": "Compresses or decompresses the contents of FlowFiles using a user-specified compression algorithm and updates the mime.type attribute as appropriate", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "org.apache.nifi.minifi.processors.CompressContent" + }, + { + "propertyDescriptors": { + "Batch Size": { + "name": "Batch Size", + "description": "The maximum number of entries processed in a single execution.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "1000" + }, + "Include Timestamp": { + "name": "Include Timestamp", + "description": "Include message timestamp in the 'timestamp' attribute.", + "validator": "BOOLEAN_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "true" + }, + "Journal Type": { + "name": "Journal Type", + "description": "Type of journal to consume.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "System", + "allowableValues": [ + { + "value": "User", + "displayName": "User" + }, + { + "value": "System", + "displayName": "System" + }, + { + "value": "Both", + "displayName": "Both" + } + ] + }, + "Payload Format": { + "name": "Payload Format", + "description": "Configures flow file content formatting. Raw: only the message. Syslog: similar to syslog or journalctl output.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Syslog", + "allowableValues": [ + { + "value": "Raw", + "displayName": "Raw" + }, + { + "value": "Syslog", + "displayName": "Syslog" + } + ] + }, + "Process Old Messages": { + "name": "Process Old Messages", + "description": "Process events created before the first usage (schedule) of the processor instance.", + "validator": "BOOLEAN_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "false" + }, + "Timestamp Format": { + "name": "Timestamp Format", + "description": "Format string to use when creating the timestamp attribute or writing messages in the syslog format. ISO/ISO 8601/ISO8601 are equivalent to \"%FT%T%Ez\". See https://howardhinnant.github.io/date/date.html#to_stream_formatting for all flags.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "%x %X %Z" + } + }, + "inputRequirement": "INPUT_FORBIDDEN", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "success", + "description": "Successfully consumed journal messages." + } + ], + "typeDescription": "Consume systemd-journald journal messages. Creates one flow file per message. Fields are mapped to attributes. Realtime timestamp is mapped to the 'timestamp' attribute. Available on Linux only.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "org.apache.nifi.minifi.extensions.systemd.ConsumeJournald" + }, + { + "propertyDescriptors": { + "Commit Offsets Policy": { + "name": "Commit Offsets Policy", + "description": "NoCommit disables offset commiting entirely. AutoCommit configures Kafka to automatically increase offsets after serving the messages. CommitAfterBatch commits offsets after the messages has been converted to flowfiles. CommitFromIncomingFlowFiles consumes incoming flowfiles and commits the offsets based on their attributes. ", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Commit After Batch", + "allowableValues": [ + { + "value": "No Commit", + "displayName": "No Commit" + }, + { + "value": "Auto Commit", + "displayName": "Auto Commit" + }, + { + "value": "Commit After Batch", + "displayName": "Commit After Batch" + }, + { + "value": "Commit from incoming flowfiles", + "displayName": "Commit from incoming flowfiles" + } + ] + }, + "Duplicate Header Handling": { + "name": "Duplicate Header Handling", + "description": "For headers to be added as attributes, this option specifies how to handle cases where multiple headers are present with the same key. For example in case of receiving these two headers: \"Accept: text/html\" and \"Accept: application/xml\" and we want to attach the value of \"Accept\" as a FlowFile attribute:\n - \"Keep First\" attaches: \"Accept -> text/html\"\n - \"Keep Latest\" attaches: \"Accept -> application/xml\"\n - \"Comma-separated Merge\" attaches: \"Accept -> text/html, application/xml\"\n", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Keep Latest", + "allowableValues": [ + { + "value": "Keep First", + "displayName": "Keep First" + }, + { + "value": "Keep Latest", + "displayName": "Keep Latest" + }, + { + "value": "Comma-separated Merge", + "displayName": "Comma-separated Merge" + } + ] + }, + "Group ID": { + "name": "Group ID", + "description": "A Group ID is used to identify consumers that are within the same consumer group. Corresponds to Kafka's 'group.id' property.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES" + }, + "Headers To Add As Attributes": { + "name": "Headers To Add As Attributes", + "description": "A comma separated list to match against all message headers. Any message header whose name matches an item from the list will be added to the FlowFile as an Attribute. If not specified, no Header values will be added as FlowFile attributes. The behaviour on when multiple headers of the same name are present is set using the Duplicate Header Handling attribute.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Honor Transactions": { + "name": "Honor Transactions", + "description": "Specifies whether or not MiNiFi should honor transactional guarantees when communicating with Kafka. If false, the Processor will use an \"isolation level\" of read_uncomitted. This means that messages will be received as soon as they are written to Kafka but will be pulled, even if the producer cancels the transactions. If this value is true, MiNiFi will not receive any messages for which the producer's transaction was canceled, but this can result in some latency since the consumer must wait for the producer to finish its entire transaction instead of pulling as the messages become available.", + "validator": "BOOLEAN_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "true" + }, + "Kafka Brokers": { + "name": "Kafka Brokers", + "description": "A comma-separated list of known Kafka Brokers in the format :.", + "validator": "NON_BLANK_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES", + "defaultValue": "localhost:9092" + }, + "Kerberos Keytab Path": { + "name": "Kerberos Keytab Path", + "description": "The path to the location on the local filesystem where the kerberos keytab is located. Read permission on the file is required.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Kerberos Principal": { + "name": "Kerberos Principal", + "description": "Kerberos Principal", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Kerberos Service Name": { + "name": "Kerberos Service Name", + "description": "Kerberos Service Name", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Key Attribute Encoding": { + "name": "Key Attribute Encoding", + "description": "FlowFiles that are emitted have an attribute named 'kafka.key'. This property dictates how the value of the attribute should be encoded.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "UTF-8", + "allowableValues": [ + { + "value": "UTF-8", + "displayName": "UTF-8" + }, + { + "value": "Hex", + "displayName": "Hex" + } + ] + }, + "Max Poll Records": { + "name": "Max Poll Records", + "description": "Specifies the maximum number of records Kafka should return when polling each time the processor is triggered.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "10000" + }, + "Max Poll Time": { + "name": "Max Poll Time", + "description": "Specifies the maximum amount of time the consumer can use for polling data from the brokers. Polling is a blocking operation, so the upper limit of this value is specified in 4 seconds.", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "4 seconds" + }, + "Message Demarcator": { + "name": "Message Demarcator", + "description": "Since KafkaConsumer receives messages in batches, you have an option to output FlowFiles which contains all Kafka messages in a single batch for a given topic and partition and this property allows you to provide a string (interpreted as UTF-8) to use for demarcating apart multiple Kafka messages. This is an optional property and if not provided each Kafka message received will result in a single FlowFile which time it is triggered. ", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES" + }, + "Message Header Encoding": { + "name": "Message Header Encoding", + "description": "Any message header that is found on a Kafka message will be added to the outbound FlowFile as an attribute. This property indicates the Character Encoding to use for deserializing the headers.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "UTF-8", + "allowableValues": [ + { + "value": "UTF-8", + "displayName": "UTF-8" + }, + { + "value": "Hex", + "displayName": "Hex" + } + ] + }, + "Offset Reset": { + "name": "Offset Reset", + "description": "Allows you to manage the condition when there is no initial offset in Kafka or if the current offset does not exist any more on the server (e.g. because that data has been deleted). Corresponds to Kafka's 'auto.offset.reset' property.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "latest", + "allowableValues": [ + { + "value": "earliest", + "displayName": "earliest" + }, + { + "value": "latest", + "displayName": "latest" + }, + { + "value": "none", + "displayName": "none" + } + ] + }, + "Password": { + "name": "Password", + "description": "The password for the given username when the SASL Mechanism is sasl_plaintext", + "validator": "VALID", + "required": "false", + "sensitive": "true", + "expressionLanguageScope": "NONE" + }, + "SASL Mechanism": { + "name": "SASL Mechanism", + "description": "The SASL mechanism to use for authentication. Corresponds to Kafka's 'sasl.mechanism' property.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "GSSAPI", + "allowableValues": [ + { + "value": "GSSAPI", + "displayName": "GSSAPI" + }, + { + "value": "PLAIN", + "displayName": "PLAIN" + } + ] + }, + "SSL Context Service": { + "typeProvidedByValue": { + "type": "org.apache.nifi.minifi.controllers.SSLContextServiceInterface", + "group": "org.apache.nifi.minifi", + "artifact": "minifi-system" + }, + "name": "SSL Context Service", + "description": "SSL Context Service Name", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Security Protocol": { + "name": "Security Protocol", + "description": "Protocol used to communicate with brokers. Corresponds to Kafka's 'security.protocol' property.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "plaintext", + "allowableValues": [ + { + "value": "plaintext", + "displayName": "plaintext" + }, + { + "value": "ssl", + "displayName": "ssl" + }, + { + "value": "sasl_plaintext", + "displayName": "sasl_plaintext" + }, + { + "value": "sasl_ssl", + "displayName": "sasl_ssl" + } + ] + }, + "Session Timeout": { + "name": "Session Timeout", + "description": "Client group session and failure detection timeout. The consumer sends periodic heartbeats to indicate its liveness to the broker. If no hearts are received by the broker for a group member within the session timeout, the broker will remove the consumer from the group and trigger a rebalance. The allowed range is configured with the broker configuration properties group.min.session.timeout.ms and group.max.session.timeout.ms.", + "validator": "TIME_PERIOD_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "60 seconds" + }, + "Topic Name Format": { + "name": "Topic Name Format", + "description": "Specifies whether the Topic(s) provided are a comma separated list of names or a single regular expression. Using regular expressions does not automatically discover Kafka topics created after the processor started.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Names", + "allowableValues": [ + { + "value": "Names", + "displayName": "Names" + }, + { + "value": "Patterns", + "displayName": "Patterns" + } + ] + }, + "Topic Names": { + "name": "Topic Names", + "description": "The name of the Kafka Topic(s) to pull from. Multiple topic names are supported as a comma separated list.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES" + }, + "Username": { + "name": "Username", + "description": "The username when the SASL Mechanism is sasl_plaintext", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + } + }, + "inputRequirement": "INPUT_ALLOWED", + "isSingleThreaded": "true", + "supportedRelationships": [ + { + "name": "success", + "description": "Incoming Kafka messages as flowfiles. Depending on the demarcation strategy, this can be one or multiple messages per flowfile." + } + ], + "typeDescription": "Consumes messages from Apache Kafka and transform them into MiNiFi FlowFiles. The application should make sure that the processor is triggered at regular intervals, even if no messages are expected, to serve any queued callbacks waiting to be called. Rebalancing can also only happen on trigger.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "true", + "type": "org.apache.nifi.minifi.processors.ConsumeKafka" + }, + { + "propertyDescriptors": { + "Add Attributes As Fields": { + "name": "Add Attributes As Fields", + "description": "If setting this property to true, default fields are going to be added in each record: _topic, _qos, _isDuplicate, _isRetained.", + "validator": "BOOLEAN_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "true" + }, + "Attribute From Content Type": { + "name": "Attribute From Content Type", + "description": "Name of FlowFile attribute to be filled from content type of received message. MQTT 5.x only.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Broker URI": { + "name": "Broker URI", + "description": "The URI to use to connect to the MQTT broker", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Clean Session": { + "name": "Clean Session", + "description": "Whether to start afresh rather than remembering previous subscriptions. If true, then make broker forget subscriptions after disconnected. MQTT 3.x only.", + "validator": "BOOLEAN_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "true" + }, + "Clean Start": { + "name": "Clean Start", + "description": "Whether to start afresh rather than remembering previous subscriptions. MQTT 5.x only.", + "validator": "BOOLEAN_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "true" + }, + "Client ID": { + "name": "Client ID", + "description": "MQTT client ID to use. If not set, a UUID will be generated.", + "validator": "NON_BLANK_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Connection Timeout": { + "name": "Connection Timeout", + "description": "Maximum time interval the client will wait for the network connection to the MQTT broker", + "validator": "TIME_PERIOD_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "10 sec" + }, + "Keep Alive Interval": { + "name": "Keep Alive Interval", + "description": "Defines the maximum time interval between messages sent or received", + "validator": "TIME_PERIOD_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "60 sec" + }, + "Last Will Content Type": { + "name": "Last Will Content Type", + "description": "Content type of the client's Last Will. MQTT 5.x only.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Last Will Message": { + "name": "Last Will Message", + "description": "The message to send as the client's Last Will. If the Last Will Message is empty, Last Will will be deleted from the broker", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Last Will QoS": { + "name": "Last Will QoS", + "description": "The Quality of Service (QoS) to send the last will with.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0", + "allowableValues": [ + { + "value": "0", + "displayName": "0" + }, + { + "value": "1", + "displayName": "1" + }, + { + "value": "2", + "displayName": "2" + } + ] + }, + "Last Will Retain": { + "name": "Last Will Retain", + "description": "Whether to retain the client's Last Will", + "validator": "BOOLEAN_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "false" + }, + "Last Will Topic": { + "name": "Last Will Topic", + "description": "The topic to send the client's Last Will to. If the Last Will topic is not set then a Last Will will not be sent", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "MQTT Version": { + "name": "MQTT Version", + "description": "The MQTT specification version when connecting to the broker.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "3.x AUTO", + "allowableValues": [ + { + "value": "3.x AUTO", + "displayName": "3.x AUTO" + }, + { + "value": "3.1.0", + "displayName": "3.1.0" + }, + { + "value": "3.1.1", + "displayName": "3.1.1" + }, + { + "value": "5.0", + "displayName": "5.0" + } + ] + }, + "Password": { + "name": "Password", + "description": "Password to use when connecting to the broker", + "validator": "VALID", + "required": "false", + "sensitive": "true", + "expressionLanguageScope": "NONE" + }, + "Quality of Service": { + "name": "Quality of Service", + "description": "The Quality of Service (QoS) of messages.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0", + "allowableValues": [ + { + "value": "0", + "displayName": "0" + }, + { + "value": "1", + "displayName": "1" + }, + { + "value": "2", + "displayName": "2" + } + ] + }, + "Queue Max Message": { + "name": "Queue Max Message", + "description": "Maximum number of messages allowed on the received MQTT queue", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "1000" + }, + "Receive Maximum": { + "name": "Receive Maximum", + "description": "Maximum number of unacknowledged messages allowed. MQTT 5.x only.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "65535" + }, + "Record Reader": { + "typeProvidedByValue": { + "type": "org.apache.nifi.minifi.core.RecordSetReader", + "group": "org.apache.nifi.minifi", + "artifact": "minifi-system" + }, + "name": "Record Reader", + "description": "The Record Reader to use for parsing received MQTT Messages into Records.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Record Writer": { + "typeProvidedByValue": { + "type": "org.apache.nifi.minifi.core.RecordSetWriter", + "group": "org.apache.nifi.minifi", + "artifact": "minifi-system" + }, + "name": "Record Writer", + "description": "The Record Writer to use for serializing Records before writing them to a FlowFile.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Security CA": { + "name": "Security CA", + "description": "File or directory path to CA certificate(s) for verifying the broker's key", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Security Cert": { + "name": "Security Cert", + "description": "Path to client's public key (PEM) used for authentication", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Security Pass Phrase": { + "name": "Security Pass Phrase", + "description": "Private key passphrase", + "validator": "VALID", + "required": "false", + "sensitive": "true", + "expressionLanguageScope": "NONE" + }, + "Security Private Key": { + "name": "Security Private Key", + "description": "Path to client's private key (PEM) used for authentication", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Security Protocol": { + "name": "Security Protocol", + "description": "Protocol used to communicate with brokers", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Session Expiry Interval": { + "name": "Session Expiry Interval", + "description": "Time to delete session on broker after client is disconnected. MQTT 5.x only.", + "validator": "TIME_PERIOD_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0 s" + }, + "Topic": { + "name": "Topic", + "description": "The topic to subscribe to.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Topic Alias Maximum": { + "name": "Topic Alias Maximum", + "description": "Maximum number of topic aliases to use. If set to 0, then topic aliases cannot be used. MQTT 5.x only.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0" + }, + "Username": { + "name": "Username", + "description": "Username to use when connecting to the broker", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + } + }, + "inputRequirement": "INPUT_FORBIDDEN", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "success", + "description": "FlowFiles that are sent successfully to the destination are transferred to this relationship" + } + ], + "typeDescription": "This Processor gets the contents of a FlowFile from a MQTT broker for a specified topic. The the payload of the MQTT message becomes content of a FlowFile. If Record Reader and Record Writer are set, then the MQTT message specific attributes are not set in the flow file, because different attributes can be set for different records. In this case if Add Attributes As Fields is set to true, the attributes will be added to each record as fields.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "org.apache.nifi.minifi.processors.ConsumeMQTT" + }, + { + "propertyDescriptors": { + "Include Zero Record FlowFiles": { + "name": "Include Zero Record FlowFiles", + "description": "When converting an incoming FlowFile, if the conversion results in no data, this property specifies whether or not a FlowFile will be sent to the corresponding relationship.", + "validator": "BOOLEAN_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "true" + }, + "Record Reader": { + "typeProvidedByValue": { + "type": "org.apache.nifi.minifi.core.RecordSetReader", + "group": "org.apache.nifi.minifi", + "artifact": "minifi-system" + }, + "name": "Record Reader", + "description": "Specifies the Controller Service to use for reading incoming data", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Record Writer": { + "typeProvidedByValue": { + "type": "org.apache.nifi.minifi.core.RecordSetWriter", + "group": "org.apache.nifi.minifi", + "artifact": "minifi-system" + }, + "name": "Record Writer", + "description": "Specifies the Controller Service to use for writing out the records", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + } + }, + "inputRequirement": "INPUT_REQUIRED", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "failure", + "description": "If a FlowFile cannot be transformed from the configured input format to the configured output format, the unchanged FlowFile will be routed to this relationship" + }, + { + "name": "success", + "description": "FlowFiles that are successfully transformed will be routed to this relationship" + } + ], + "typeDescription": "Converts records from one data format to another using configured Record Reader and Record Writer Controller Services.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "org.apache.nifi.minifi.processors.ConvertRecord" + }, + { + "propertyDescriptors": { + "Max Buffer Age": { + "name": "Max Buffer Age", + "description": "The maximum age of the buffer after which it will be transferred to success when matching Start of Message patterns or to failure when matching End of Message patterns. Expected format is