公网SASL_SSL方式生产和消费消息

1yum install librdkafka-devel
2yum install cyrus-sasl
3yum install cyrus-sasl-scram

1/*
2* librdkafka - Apache Kafka C library
3*
4* Copyright (c) 2017, Magnus Edenhill
5* All rights reserved.
6*
7* Redistribution and use in source and binary forms, with or without
8* modification, are permitted provided that the following conditions are met:
9*
10* 1. Redistributions of source code must retain the above copyright notice,
11*    this list of conditions and the following disclaimer.
12* 2. Redistributions in binary form must reproduce the above copyright notice,
13*    this list of conditions and the following disclaimer in the documentation
14*    and/or other materials provided with the distribution.
15*
16* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26* POSSIBILITY OF SUCH DAMAGE.
27  */
28
29/**
30* Simple Apache Kafka producer
31* using the Kafka driver from librdkafka
32* (https://github.com/edenhill/librdkafka)
33  */
34
35#include <stdio.h>
36#include <signal.h>
37#include <string.h>
38
39
40/* Typical include path would be <librdkafka/rdkafka.h>, but this program
41* is builtin from within the librdkafka source tree and thus differs. */
42  #include "librdkafka/rdkafka.h"
43
44
45static volatile sig_atomic_t run = 1;
46
47/**
48* @brief Signal termination of program
49  */
50  static void stop(int sig) {
51  run = 0;
52  fclose(stdin); /* abort fgets() */
53  }
54
55
56/**
57* @brief Message delivery report callback.
58*
59* This callback is called exactly once per message, indicating if
60* the message was succesfully delivered
61* (rkmessage->err == RD_KAFKA_RESP_ERR_NO_ERROR) or permanently
62* failed delivery (rkmessage->err != RD_KAFKA_RESP_ERR_NO_ERROR).
63*
64* The callback is triggered from rd_kafka_poll() and executes on
65* the application's thread.
66  */
67  static void dr_msg_cb(rd_kafka_t *rk, const rd_kafka_message_t *rkmessage, void *opaque) {
68  if (rkmessage->err) {
69  fprintf(stderr, "%% Message delivery failed: %s\n", rd_kafka_err2str(rkmessage->err));
70  } else {
71  fprintf(stderr, "%% Message delivered (%zd bytes, partition %" PRId32 ")\n", rkmessage->len, rkmessage->partition);
72  }
73
74  /* The rkmessage is destroyed automatically by librdkafka */
75  }
76
77
78
79int main(int argc, char **argv) {
80rd_kafka_t *rk;        /* Producer instance handle */
81rd_kafka_conf_t *conf; /* Temporary configuration object */
82
83    char errstr[512];      /* librdkafka API error reporting buffer */
84    char buf[512];         /* Message value temporary buffer */
85
86    const char *brokers;   /* Argument: broker list */
87    const char *topic;     /* Argument: topic to produce to */
88    const char *username;      /* Argument: sasl username */
89    const char *password;      /* Argument: sasl password */
90
91    /*
92     * Argument validation
93     */
94     
95    //检查参数配置
96    if (argc != 5) {
97        fprintf(stderr, "%% Usage: %s <broker> <topic> <username> <password>\n", argv[0]);
98        return 1;
99    }
100
101    // 接入点信息
102    brokers = argv[1];
103    // 主题名称
104    topic = argv[2];
105    // 用户管理中创建的用户名称
106    username = argv[3];
107    // 用户管理中创建的用户密码
108    password = argv[4];
109
110
111    /*
112     * Create Kafka client configuration place-holder
113     */
114    conf = rd_kafka_conf_new();
115
116    /* Set bootstrap broker(s) as a comma-separated list of
117     * host or host:port (default port 9092).
118     * librdkafka will use the bootstrap brokers to acquire the full
119     * set of brokers from the cluster. */
120    if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) {
121        fprintf(stderr, "%s\n", errstr);
122        return 1;
123    }
124    // 认证机制支持PLAIN、SCRAM-SHA-512两种机制，根据集群所使用的认证方式进行选择
125    if (
126            rd_kafka_conf_set(conf, "ssl.ca.location", "ca.pem", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
127            || rd_kafka_conf_set(conf, "security.protocol", "sasl_ssl", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
128            || rd_kafka_conf_set(conf, "sasl.mechanism", "SCRAM-SHA-512", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
129            || rd_kafka_conf_set(conf, "sasl.username", username, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
130            || rd_kafka_conf_set(conf, "sasl.password", password, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
131    ) {
132        fprintf(stderr, "%s\n", errstr);
133        return -1;
134    }
135
136    /* Set the delivery report callback.
137     * This callback will be called once per message to inform
138     * the application if delivery succeeded or failed.
139     * See dr_msg_cb() above.
140     * The callback is only triggered from rd_kafka_poll() and
141     * rd_kafka_flush(). */
142    rd_kafka_conf_set_dr_msg_cb(conf, dr_msg_cb);
143
144    /*
145     * Create producer instance.
146     *
147     * NOTE: rd_kafka_new() takes ownership of the conf object
148     *       and the application must not reference it again after
149     *       this call.
150     */
151    rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr));
152    if (!rk) {
153        fprintf(stderr, "%% Failed to create new producer: %s\n", errstr);
154        return 1;
155    }
156
157    /* Signal handler for clean shutdown */
158    signal(SIGINT, stop);
159
160    fprintf(stderr,
161            "%% Type some text and hit enter to produce message\n"
162            "%% Or just hit enter to only serve delivery reports\n"
163            "%% Press Ctrl-C or Ctrl-D to exit\n");
164
165    while (run && fgets(buf, sizeof(buf), stdin)) {
166        size_t len = strlen(buf);
167        rd_kafka_resp_err_t err;
168
169        /* Remove newline */
170        if (buf[len - 1] == '\n') {
171            buf[--len] = '\0';
172        }
173
174        /* Empty line: only serve delivery reports */
175        if (len == 0) {
176            rd_kafka_poll(rk, 0/*non-blocking */);
177            continue;
178        }
179
180        /*
181         * Send/Produce message.
182         * This is an asynchronous call, on success it will only
183         * enqueue the message on the internal producer queue.
184         * The actual delivery attempts to the broker are handled
185         * by background threads.
186         * The previously registered delivery report callback
187         * (dr_msg_cb) is used to signal back to the application
188         * when the message has been delivered (or failed).
189         */
190    retry:
191        err = rd_kafka_producev(
192            /* Producer handle */
193            rk,
194            /* Topic name */
195            RD_KAFKA_V_TOPIC(topic),
196            /* Make a copy of the payload. */
197            RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY),
198            /* Message value and length */
199            RD_KAFKA_V_VALUE(buf, len),
200            /* Per-Message opaque, provided in
201             * delivery report callback as
202             * msg_opaque. */
203            RD_KAFKA_V_OPAQUE(NULL),
204            /* End sentinel */
205            RD_KAFKA_V_END
206        );
207
208        if (err) {
209            /*
210             * Failed to *enqueue* message for producing.
211             */
212            fprintf(stderr,
213                    "%% Failed to produce to topic %s: %s\n", topic,
214                    rd_kafka_err2str(err));
215
216            if (err == RD_KAFKA_RESP_ERR__QUEUE_FULL) {
217                /* If the internal queue is full, wait for
218                 * messages to be delivered and then retry.
219                 * The internal queue represents both
220                 * messages to be sent and messages that have
221                 * been sent or failed, awaiting their
222                 * delivery report callback to be called.
223                 *
224                 * The internal queue is limited by the
225                 * configuration property
226                 * queue.buffering.max.messages */
227                rd_kafka_poll(rk, 1000 /*block for max 1000ms*/);
228                goto retry;
229            }
230        } else {
231            fprintf(stderr, "%% Enqueued message (%zd bytes) "
232                    "for topic %s\n",
233                    len, topic);
234        }
235
236
237        /* A producer application should continually serve
238         * the delivery report queue by calling rd_kafka_poll()
239         * at frequent intervals.
240         * Either put the poll call in your main loop, or in a
241         * dedicated thread, or call it after every
242         * rd_kafka_produce() call.
243         * Just make sure that rd_kafka_poll() is still called
244         * during periods where you are not producing any messages
245         * to make sure previously produced messages have their
246         * delivery report callback served (and any other callbacks
247         * you register). */
248        rd_kafka_poll(rk, 0 /*non-blocking*/);
249    }
250
251
252    /* Wait for final messages to be delivered or fail.
253     * rd_kafka_flush() is an abstraction over rd_kafka_poll() which
254     * waits for all messages to be delivered. */
255    fprintf(stderr, "%% Flushing final messages..\n");
256    rd_kafka_flush(rk, 10 * 1000 /* wait for max 10 seconds */);
257
258    /* If the output queue is still not empty there is an issue
259     * with producing messages to the clusters. */
260    if (rd_kafka_outq_len(rk) > 0) {
261        fprintf(stderr, "%% %d message(s) were not delivered\n", rd_kafka_outq_len(rk));
262    }
263
264    /* Destroy the producer instance */
265    rd_kafka_destroy(rk);
266
267    return 0;
268}

1/*
2* librdkafka - Apache Kafka C library
3*
4* Copyright (c) 2019, Magnus Edenhill
5* All rights reserved.
6*
7* Redistribution and use in source and binary forms, with or without
8* modification, are permitted provided that the following conditions are met:
9*
10* 1. Redistributions of source code must retain the above copyright notice,
11*    this list of conditions and the following disclaimer.
12* 2. Redistributions in binary form must reproduce the above copyright notice,
13*    this list of conditions and the following disclaimer in the documentation
14*    and/or other materials provided with the distribution.
15*
16* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26* POSSIBILITY OF SUCH DAMAGE.
27  */
28
29/**
30* Simple high-level balanced Apache Kafka consumer
31* using the Kafka driver from librdkafka
32* (https://github.com/edenhill/librdkafka)
33  */
34
35#include <stdio.h>
36#include <signal.h>
37#include <string.h>
38#include <ctype.h>
39
40
41/* Typical include path would be <librdkafka/rdkafka.h>, but this program
42* is builtin from within the librdkafka source tree and thus differs. */
43  //#include <librdkafka/rdkafka.h>
44  #include "librdkafka/rdkafka.h"
45
46
47static volatile sig_atomic_t run = 1;
48
49/**
50* @brief Signal termination of program
51  */
52  static void stop(int sig) {
53  run = 0;
54  }
55
56
57
58/**
59* @returns 1 if all bytes are printable, else 0.
60  */
61  static int is_printable(const char *buf, size_t size) {
62  size_t i;
63
64  for (i = 0; i < size; i++) {
65  if (!isprint((int)buf[i])) {
66  return 0;
67  }
68  }
69
70  return 1;
71  }
72
73
74int main(int argc, char **argv) {
75rd_kafka_t *rk;          /* Consumer instance handle */
76rd_kafka_conf_t *conf;   /* Temporary configuration object */
77rd_kafka_resp_err_t err; /* librdkafka API error code */
78
79    char errstr[512];        /* librdkafka API error reporting buffer */
80
81    const char *brokers;     /* Argument: broker list */
82    const char *groupid;     /* Argument: Consumer group id */
83    const char *username;      /* Argument: sasl username */
84    const char *password;      /* Argument: sasl password */
85    char **topics;           /* Argument: list of topics to subscribe to */
86
87    int topic_cnt;           /* Number of topics to subscribe to */
88    rd_kafka_topic_partition_list_t *subscription; /* Subscribed topics */
89    int i;
90
91    /*
92     * Argument validation
93     */
94    if (argc < 6) {
95        fprintf(stderr, "%% Usage: %s <broker> <group.id> <username> <password> <topic1> <topic2>..\n", argv[0]);
96        return 1;
97    }
98
99    brokers   = argv[1];
100    groupid   = argv[2];
101    username  = argv[3];
102    password  = argv[4];
103    topics    = &argv[5];
104
105    topic_cnt = argc - 5;
106
107
108    /*
109     * Create Kafka client configuration place-holder
110     */
111    conf = rd_kafka_conf_new();
112
113    /* Set bootstrap broker(s) as a comma-separated list of
114     * host or host:port (default port 9092).
115     * librdkafka will use the bootstrap brokers to acquire the full
116     * set of brokers from the cluster. */
117    if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) {
118        fprintf(stderr, "%s\n", errstr);
119        rd_kafka_conf_destroy(conf);
120        return 1;
121    }
122    // 认证机制支持PLAIN、SCRAM-SHA-512两种机制，根据集群所使用的认证方式进行选择
123    if (
124            rd_kafka_conf_set(conf, "ssl.ca.location", "ca.pem", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
125            || rd_kafka_conf_set(conf, "security.protocol", "sasl_ssl", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
126            || rd_kafka_conf_set(conf, "sasl.mechanism", "SCRAM-SHA-512", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
127            || rd_kafka_conf_set(conf, "sasl.username", username, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
128            || rd_kafka_conf_set(conf, "sasl.password", password, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
129    ) {
130        fprintf(stderr, "%s\n", errstr);
131        return -1;
132    }
133
134    /* Set the consumer group id.
135     * All consumers sharing the same group id will join the same
136     * group, and the subscribed topic' partitions will be assigned
137     * according to the partition.assignment.strategy
138     * (consumer config property) to the consumers in the group. */
139    if (rd_kafka_conf_set(conf, "group.id", groupid, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) {
140        fprintf(stderr, "%s\n", errstr);
141        rd_kafka_conf_destroy(conf);
142        return 1;
143    }
144
145    /* If there is no previously committed offset for a partition
146     * the auto.offset.reset strategy will be used to decide where
147     * in the partition to start fetching messages.
148     * By setting this to earliest the consumer will read all messages
149     * in the partition if there was no previously committed offset. */
150    if (rd_kafka_conf_set(conf, "auto.offset.reset", "earliest", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) {
151        fprintf(stderr, "%s\n", errstr);
152        rd_kafka_conf_destroy(conf);
153        return 1;
154    }
155
156    /*
157     * Create consumer instance.
158     *
159     * NOTE: rd_kafka_new() takes ownership of the conf object
160     *       and the application must not reference it again after
161     *       this call.
162     */
163    rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr));
164    if (!rk) {
165        fprintf(stderr, "%% Failed to create new consumer: %s\n", errstr);
166        return 1;
167    }
168
169    conf = NULL; /* Configuration object is now owned, and freed,
170                  * by the rd_kafka_t instance. */
171
172
173    /* Redirect all messages from per-partition queues to
174     * the main queue so that messages can be consumed with one
175     * call from all assigned partitions.
176     *
177     * The alternative is to poll the main queue (for events)
178     * and each partition queue separately, which requires setting
179     * up a rebalance callback and keeping track of the assignment:
180     * but that is more complex and typically not recommended. */
181    rd_kafka_poll_set_consumer(rk);
182
183
184    /* Convert the list of topics to a format suitable for librdkafka */
185    subscription = rd_kafka_topic_partition_list_new(topic_cnt);
186    for (i = 0; i < topic_cnt; i++) {
187        rd_kafka_topic_partition_list_add(
188            subscription,
189            topics[i],
190            /* the partition is ignored
191             * by subscribe() */
192            RD_KAFKA_PARTITION_UA
193        );
194    }
195
196    /* Subscribe to the list of topics */
197    err = rd_kafka_subscribe(rk, subscription);
198    if (err) {
199        fprintf(stderr, "%% Failed to subscribe to %d topics: %s\n", subscription->cnt, rd_kafka_err2str(err));
200        rd_kafka_topic_partition_list_destroy(subscription);
201        rd_kafka_destroy(rk);
202        return 1;
203    }
204
205    fprintf(stderr,
206            "%% Subscribed to %d topic(s), "
207            "waiting for rebalance and messages...\n",
208            subscription->cnt);
209
210    rd_kafka_topic_partition_list_destroy(subscription);
211
212
213    /* Signal handler for clean shutdown */
214    signal(SIGINT, stop);
215
216    /* Subscribing to topics will trigger a group rebalance
217     * which may take some time to finish, but there is no need
218     * for the application to handle this idle period in a special way
219     * since a rebalance may happen at any time.
220     * Start polling for messages. */
221
222    while (run) {
223        rd_kafka_message_t *rkm;
224
225        rkm = rd_kafka_consumer_poll(rk, 100);
226        if (!rkm) {
227            continue; /* Timeout: no message within 100ms,
228                       *  try again. This short timeout allows
229                       *  checking for `run` at frequent intervals.
230                       */
231        }
232
233        /* consumer_poll() will return either a proper message
234         * or a consumer error (rkm->err is set). */
235        if (rkm->err) {
236            /* Consumer errors are generally to be considered
237             * informational as the consumer will automatically
238             * try to recover from all types of errors. */
239            fprintf(stderr, "%% Consumer error: %s\n", rd_kafka_message_errstr(rkm));
240            rd_kafka_message_destroy(rkm);
241            continue;
242        }
243
244        /* Proper message. */
245        printf("Message on %s [%" PRId32 "] at offset %" PRId64 ":\n",
246               rd_kafka_topic_name(rkm->rkt), rkm->partition,
247               rkm->offset);
248
249        /* Print the message key. */
250        if (rkm->key && is_printable(rkm->key, rkm->key_len)) {
251            printf(" Key: %.*s\n", (int)rkm->key_len, (const char *)rkm->key);
252        } else if (rkm->key) {
253            printf(" Key: (%d bytes)\n", (int)rkm->key_len);
254        }
255
256        /* Print the message value/payload. */
257        if (rkm->payload && is_printable(rkm->payload, rkm->len)) {
258            printf(" Value: %.*s\n", (int)rkm->len, (const char *)rkm->payload);
259        } else if (rkm->payload) {
260            printf(" Value: (%d bytes)\n", (int)rkm->len);
261        }
262
263        rd_kafka_message_destroy(rkm);
264    }
265
266
267    /* Close the consumer: commit final offsets and leave the group. */
268    fprintf(stderr, "%% Closing consumer\n");
269    rd_kafka_consumer_close(rk);
270
271
272    /* Destroy the consumer */
273    rd_kafka_destroy(rk);
274
275    return 0;
276}

1# 启动消费者
2gcc -lrdkafka ./consumer.c -o consumer
3./consumer <broker> <group.id> <username> <password> <topic1> <topic2>..
4# 启动生产者
5gcc -lrdkafka ./producer.c -o producer
6./producer <broker> <topic> <username> <password>