1 #include <stdio.h>
  2 #include <stdlib.h>
  3 #include <string.h>
  4 #include <stdint.h>
  5 #include <inttypes.h>
  6 #include <sys/types.h>
  7 #include <sys/queue.h>
  8 #include <netinet/in.h>
  9 #include <setjmp.h>
 10 #include <stdarg.h>
 11 #include <ctype.h>
 12 #include <errno.h>
 13 #include <getopt.h>
 14 
 15 #include <rte_common.h>
 16 #include <rte_log.h>
 17 #include <rte_memory.h>
 18 #include <rte_memcpy.h>
 19 #include <rte_memzone.h>
 20 #include <rte_eal.h>
 21 #include <rte_per_lcore.h>
 22 #include <rte_launch.h>
 23 #include <rte_atomic.h>
 24 #include <rte_cycles.h>
 25 #include <rte_prefetch.h>
 26 #include <rte_lcore.h>
 27 #include <rte_per_lcore.h>
 28 #include <rte_branch_prediction.h>
 29 #include <rte_interrupts.h>
 30 #include <rte_pci.h>
 31 #include <rte_random.h>
 32 #include <rte_debug.h>
 33 #include <rte_ether.h>
 34 #include <rte_ethdev.h>
 35 #include <rte_ring.h>
 36 #include <rte_mempool.h>
 37 #include <rte_mbuf.h>
 38 
 39 #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1
 40 
 41 #define MBUF_SIZE (2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM)
 42 #define NB_MBUF   8192
 43 
 44 #define MAX_PKT_BURST 32
 45 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
 46 
 47 /*
 48  * Configurable number of RX/TX ring descriptors
 49  */
 50 #define RTE_TEST_RX_DESC_DEFAULT 128
 51 #define RTE_TEST_TX_DESC_DEFAULT 512
 52 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
 53 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
 54 
 55 /*物理埠的mac地址的陣列       ethernet addresses of ports */
 56 static struct ether_addr l2fwd_ports_eth_addr[RTE_MAX_ETHPORTS];
 57 
 58 /*已經啟用的物理埠的掩碼/點陣圖    mask of enabled ports */
 59 static uint32_t l2fwd_enabled_port_mask = 0;
 60 
 61 /*已經啟用的目的物理埠編號的陣列    list of enabled ports */
 62 static uint32_t l2fwd_dst_ports[RTE_MAX_ETHPORTS];
 63 
 64 static unsigned int l2fwd_rx_queue_per_lcore = 1; //預設值，每個lcore負責的接收佇列數量
 65 
 66 struct mbuf_table {  //mbuf陣列，可以存放32個數據包
 67     unsigned len;
 68     struct rte_mbuf *m_table[MAX_PKT_BURST];
 69 };
 70 
 71 #define MAX_RX_QUEUE_PER_LCORE 16
 72 #define MAX_TX_QUEUE_PER_PORT 16
 73 struct lcore_queue_conf {
 74     unsigned n_rx_port;  //用於接收資料包的物理埠的實際數量
 75     unsigned rx_port_list[MAX_RX_QUEUE_PER_LCORE];
 76     struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS]; //儲存傳送資料包的快取區
 77 
 78 } __rte_cache_aligned;
 79 struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE];
 80 
 81 static const struct rte_eth_conf port_conf = {
 82     .rxmode = {
 83         .split_hdr_size = 0,
 84         .header_split   = 0, /**< Header Split disabled */
 85         .hw_ip_checksum = 0, /**< IP checksum offload disabled */
 86         .hw_vlan_filter = 0, /**< VLAN filtering disabled */
 87         .jumbo_frame    = 0, /**< Jumbo Frame Support disabled */
 88         .hw_strip_crc   = 0, /**< CRC stripped by hardware */
 89     },
 90     .txmode = {
 91         .mq_mode = ETH_MQ_TX_NONE,
 92     },
 93 };
 94 
 95 struct rte_mempool * l2fwd_pktmbuf_pool = NULL;
 96 
 97 /*每個物理埠的統計結構體   Per-port statistics struct */
 98 struct l2fwd_port_statistics {
 99     uint64_t tx;
100     uint64_t rx;
101     uint64_t dropped;
102 } __rte_cache_aligned;
103 struct l2fwd_port_statistics port_statistics[RTE_MAX_ETHPORTS]; //資料包的統計資訊的全域性陣列
104 
105 /* A tsc-based timer responsible for triggering statistics printout */
106 #define TIMER_MILLISECOND 2000000ULL /* around 1ms at 2 Ghz */
107 #define MAX_TIMER_PERIOD 86400 /* 1 day max */
108 static int64_t timer_period = 10 * TIMER_MILLISECOND * 1000; /* default period is 10 seconds */
109 
110 /* Print out statistics on packets dropped */
111 static void //列印資料包丟失等統計資訊
112 print_stats(void)
113 {
114     uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
115     unsigned portid;
116 
117     total_packets_dropped = 0;
118     total_packets_tx = 0;
119     total_packets_rx = 0;
120 
121     const char clr[] = { 27, '[', '2', 'J', '\0' };
122     const char topLeft[] = { 27, '[', '1', ';', '1', 'H','\0' };
123 
124         /* Clear screen and move to top left */
125     printf("%s%s", clr, topLeft);
126 
127     printf("\nPort statistics ====================================");
128 
129     for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
130         /* skip disabled ports */
131         if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
132             continue;
133         printf("\nStatistics for port %u ------------------------------"
134                "\nPackets sent: %24"PRIu64
135                "\nPackets received: %20"PRIu64
136                "\nPackets dropped: %21"PRIu64,
137                portid,
138                port_statistics[portid].tx,
139                port_statistics[portid].rx,
140                port_statistics[portid].dropped);
141 
142         total_packets_dropped += port_statistics[portid].dropped;
143         total_packets_tx += port_statistics[portid].tx;
144         total_packets_rx += port_statistics[portid].rx;
145     }
146     printf("\nAggregate statistics ==============================="
147            "\nTotal packets sent: %18"PRIu64
148            "\nTotal packets received: %14"PRIu64
149            "\nTotal packets dropped: %15"PRIu64,
150            total_packets_tx,
151            total_packets_rx,
152            total_packets_dropped);
153     printf("\n====================================================\n");
154 }
155 
156 /* Send the burst of packets on an output interface */
157 static int  //在一個輸出介面上burst傳送資料包
158 l2fwd_send_burst(struct lcore_queue_conf *qconf, unsigned n, uint8_t port)
159 {
160     struct rte_mbuf **m_table;
161     unsigned ret;
162     unsigned queueid =0;
163 
164     m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table;
165                 //burst輸出資料包
166     ret = rte_eth_tx_burst(port, (uint16_t) queueid, m_table, (uint16_t) n);
167     port_statistics[port].tx += ret; //記錄發包數量
168     if (unlikely(ret < n)) {
169         port_statistics[port].dropped += (n - ret); //記錄丟包數量
170         do {
171             rte_pktmbuf_free(m_table[ret]);
172         } while (++ret < n);
173     }
174 
175     return 0;
176 }
177 
178 /* Enqueue packets for TX and prepare them to be sent */
179 static int  //把資料包入隊到傳送緩衝區
180 l2fwd_send_packet(struct rte_mbuf *m, uint8_t port)
181 {
182     unsigned lcore_id, len;
183     struct lcore_queue_conf *qconf;
184 
185     lcore_id = rte_lcore_id(); //取得正在執行的lcore編號
186 
187     qconf = &lcore_queue_conf[lcore_id];//取得lcore_queue的配置
188     len = qconf->tx_mbufs[port].len; //得到發包快取區中資料包的個數
189     qconf->tx_mbufs[port].m_table[len] = m;//指向資料包
190     len++;
191 
192     /* enough pkts to be sent */
193     if (unlikely(len == MAX_PKT_BURST)) { //如果累計到32個數據包
194         l2fwd_send_burst(qconf, MAX_PKT_BURST, port); //實際傳送資料包
195         len = 0;
196     }
197 
198     qconf->tx_mbufs[port].len = len;//更新發包快取區中的資料包的個數
199     return 0;
200 }
201 
202 static void
203 l2fwd_simple_forward(struct rte_mbuf *m, unsigned portid)
204 { 
205    
206 
207     struct ether_hdr *eth;
208     void *tmp;
209     unsigned dst_port;
210 
211     dst_port = l2fwd_dst_ports[portid]; 
212     eth = rte_pktmbuf_mtod(m, struct ether_hdr *);
213    
214     /* 02:00:00:00:00:xx  修改目的mac地址 */
215     tmp = &eth->d_addr.addr_bytes[0];
216     *((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dst_port << 40);
217 
218     /* src addr  修改進入包的目的mac地址為轉發包的源mac地址  */
219     ether_addr_copy(&l2fwd_ports_eth_addr[dst_port], &eth->s_addr);
220 
221     l2fwd_send_packet(m, (uint8_t) dst_port); //在dst_port上傳送資料包
222 }
223 
224 /* main processing loop */
225 static void  //執行緒的主處理迴圈
226 l2fwd_main_loop(void)
227 {
228     struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
229     struct rte_mbuf *m;
230     unsigned lcore_id;
231     uint64_t prev_tsc, diff_tsc, cur_tsc, timer_tsc;
232     unsigned i, j, portid, nb_rx;
233     struct lcore_queue_conf *qconf;
234     const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US;
235 
236     prev_tsc = 0;
237     timer_tsc = 0;
238 
239     lcore_id = rte_lcore_id(); //獲取當期lcore的編號
240     qconf = &lcore_queue_conf[lcore_id]; //讀取此lcore上的配置資訊
241 
242     if (qconf->n_rx_port == 0) {  //如果此lcore上的用於接收的物理埠數量為0
243         RTE_LOG(INFO, L2FWD, "lcore %u has nothing to do\n", lcore_id);
244         return; //那麼結束該執行緒
245     }
246 
247     RTE_LOG(INFO, L2FWD, "entering main loop on lcore %u\n", lcore_id);
248 
249     for (i = 0; i < qconf->n_rx_port; i++) {  //遍歷所有的用於接收資料包的物理埠
250 
251         portid = qconf->rx_port_list[i];//一個lcore可能負責多個接收用的物理埠
252         RTE_LOG(INFO, L2FWD, " -- lcoreid=%u portid=%u\n", lcore_id,
253             portid);
254     }
255 
256     while (1) { //死迴圈
257 
258         cur_tsc = rte_rdtsc();
259 
260         /*
261          * TX burst queue drain
262          */
263         diff_tsc = cur_tsc - prev_tsc; 
264         if (unlikely(diff_tsc > drain_tsc)) {
265 
266             for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
267                 if (qconf->tx_mbufs[portid].len == 0)
268                     continue;
269                 l2fwd_send_burst(&lcore_queue_conf[lcore_id],
270                          qconf->tx_mbufs[portid].len,
271                          (uint8_t) portid);
272                 qconf->tx_mbufs[portid].len = 0;
273             }
274 
275             /* if timer is enabled */
276             if (timer_period > 0) { //如果定時器啟動
277 
278                 /* advance the timer */
279                 timer_tsc += diff_tsc;
280 
281                 /* if timer has reached its timeout */
282                 if (unlikely(timer_tsc >= (uint64_t) timer_period)) {
283 
284                     /* do this only on master core */
285                     if (lcore_id == rte_get_master_lcore()) {
286                         print_stats();  //十秒鐘列印一次收包統計資訊
287                         /* reset the timer */
288                         timer_tsc = 0;
289                     }
290                 }
291             }
292 
293             prev_tsc = cur_tsc;
294         }
295 
296         /*
297          * Read packet from RX queues
298          */
299         for (i = 0; i < qconf->n_rx_port; i++) {  //遍歷所有的用於接收資料包的物理埠
300 
301             portid = qconf->rx_port_list[i]; //第i個物理埠
302             nb_rx = rte_eth_rx_burst((uint8_t) portid, 0, //接收資料包，返回實際個數
303                          pkts_burst, MAX_PKT_BURST);
304 
305             port_statistics[portid].rx += nb_rx;  //記錄物理埠上收包數量
306 
307             for (j = 0; j < nb_rx; j++) { //遍歷實際接收到的所有的資料包
308                 m = pkts_burst[j];
309                 rte_prefetch0(rte_pktmbuf_mtod(m, void *)); //預取
310                 l2fwd_simple_forward(m, portid);//簡單的二層轉發資料包
311             }
312         }
313     }
314 }
315 
316 static int
317 l2fwd_launch_one_lcore(__attribute__((unused)) void *dummy)
318 {
319     l2fwd_main_loop();//執行緒執行函式
320     return 0;
321 }
322 
323 /* display usage */
324 static void
325 l2fwd_usage(const char *prgname)
326 {
327     printf("%s [EAL options] -- -p PORTMASK [-q NQ]\n"
328            "  -p PORTMASK: hexadecimal bitmask of ports to configure\n"
329            "  -q NQ: number of queue (=ports) per lcore (default is 1)\n"
330            "  -T PERIOD: statistics will be refreshed each PERIOD seconds (0 to disable, 10 default, 86400 maximum)\n",
331            prgname);
332 }
333 
334 static int
335 l2fwd_parse_portmask(const char *portmask)
336 {
337     char *end = NULL;
338     unsigned long pm;
339 
340     /* parse hexadecimal string */
341     pm = strtoul(portmask, &end, 16);
342     if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0'))
343         return -1;
344 
345     if (pm == 0)
346         return -1;
347 
348     return pm;
349 }
350 
351 static unsigned int
352 l2fwd_parse_nqueue(const char *q_arg)
353 {
354     char *end = NULL;
355     unsigned long n;
356 
357     /* parse hexadecimal string */
358     n = strtoul(q_arg, &end, 10); //轉換為十進位制
359     if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
360         return 0;
361     if (n == 0)
362         return 0;
363     if (n >= MAX_RX_QUEUE_PER_LCORE)
364         return 0;
365 
366     return n;
367 }
368 
369 static int
370 l2fwd_parse_timer_period(const char *q_arg)
371 {
372     char *end = NULL;
373     int n;
374 
375     /* parse number string */
376     n = strtol(q_arg, &end, 10); //轉換為十進位制
377     if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
378         return -1;
379     if (n >= MAX_TIMER_PERIOD)
380         return -1;
381 
382     return n;
383 }
384 
385 /* Parse the argument given in the command line of the application */
386 static int
387 l2fwd_parse_args(int argc, char **argv)
388 {
389     int opt, ret;
390     char **argvopt;
391     int option_index;
392     char *prgname = argv[0];
393     static struct option lgopts[] = {
394         {NULL, 0, 0, 0}
395     };
396 
397     argvopt = argv;
398 
399     while ((opt = getopt_long(argc, argvopt, "p:q:T:",
400                   lgopts, &option_index)) != EOF) {
401 
402         switch (opt) {
403         /* portmask */
404         case 'p': //物理埠的掩碼
405             l2fwd_enabled_port_mask = l2fwd_parse_portmask(optarg);
406             if (l2fwd_enabled_port_mask == 0) {
407                 printf("invalid portmask\n");
408                 l2fwd_usage(prgname);
409                 return -1;
410             }
411             break;
412 
413         /* nqueue */
414         case 'q':  //lcore負責的佇列的數量
415             l2fwd_rx_queue_per_lcore = l2fwd_parse_nqueue(optarg);//修改預設值
416             if (l2fwd_rx_queue_per_lcore == 0) {
417                 printf("invalid queue number\n");
418                 l2fwd_usage(prgname);
419                 return -1;
420             }
421             break;
422 
423         /* timer period */
424         case 'T': //定時的長度
425             timer_period = l2fwd_parse_timer_period(optarg) * 1000 * TIMER_MILLISECOND;
426             if (timer_period < 0) {
427                 printf("invalid timer period\n");
428                 l2fwd_usage(prgname);
429                 return -1;
430             }
431             break;
432 
433         /* long options */
434         case 0:
435             l2fwd_usage(prgname);
436             return -1;
437 
438         default:
439             l2fwd_usage(prgname);
440             return -1;
441         }
442     }
443 
444     if (optind >= 0)
445         argv[optind-1] = prgname;
446 
447     ret = optind-1;
448     optind = 0; /* reset getopt lib */
449     return ret;
450 }
451 
452 /* Check the link status of all ports in up to 9s, and print them finally */
453 static void //檢查物理埠的連線狀態
454 check_all_ports_link_status(uint8_t port_num, uint32_t port_mask)
455 {
456 #define CHECK_INTERVAL 100 /* 100ms */
457 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
458     uint8_t portid, count, all_ports_up, print_flag = 0;
459     struct rte_eth_link link;
460 
461     printf("\nChecking link status");
462     fflush(stdout);
463     for (count = 0; count <= MAX_CHECK_TIME; count++) {
464         all_ports_up = 1;
465         for (portid = 0; portid < port_num; portid++) {
466             if ((port_mask & (1 << portid)) == 0)
467                 continue;
468             memset(&link, 0, sizeof(link));
469             rte_eth_link_get_nowait(portid, &link);
470             /* print link status if flag set */
471             if (print_flag == 1) {
472                 if (link.link_status)
473                     printf("Port %d Link Up - speed %u "
474                         "Mbps - %s\n", (uint8_t)portid,
475                         (unsigned)link.link_speed,
476                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
477                     ("full-duplex") : ("half-duplex\n"));
478                 else
479                     printf("Port %d Link Down\n",
480                         (uint8_t)portid);
481                 continue;
482             }
483             /* clear all_ports_up flag if any link down */
484             if (link.link_status == 0) {
485                 all_ports_up = 0;
486                 break;
487             }
488         }
489         /* after finally printing all link status, get out */
490         if (print_flag == 1)
491             break;
492 
493         if (all_ports_up == 0) {
494             printf(".");
495             fflush(stdout);
496             rte_delay_ms(CHECK_INTERVAL);
497         }
498 
499         /* set the print_flag if all ports up or timeout */
500         if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
501             print_flag = 1;
502             printf("done\n");
503         }
504     }
505 }
506 
507 int  //主函式
508 main(int argc, char **argv)
509 {
510     struct lcore_queue_conf *qconf;
511     struct rte_eth_dev_info dev_info;
512     int ret;
513     uint8_t nb_ports;
514     uint8_t nb_ports_available;
515     uint8_t portid, last_port;
516     unsigned lcore_id, rx_lcore_id;
517     unsigned nb_ports_in_mask = 0;
518 
519     /* init EAL */
520     ret = rte_eal_init(argc, argv); //初始化環境抽象層，並解析相關引數
521     if (ret < 0)
522         rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
523     argc -= ret;
524     argv += ret;
525 
526     /* parse application arguments (after the EAL ones) */
527     ret = l2fwd_parse_args(argc, argv); //解析l2fwd相關的引數: -p -q -P
528     if (ret < 0)
529         rte_exit(EXIT_FAILURE, "Invalid L2FWD arguments\n");
530 
531     /* create the mbuf pool */
532     l2fwd_pktmbuf_pool =       //建立mbuf pool
533         rte_mempool_create("mbuf_pool", NB_MBUF,
534                    MBUF_SIZE, 32,
535                    sizeof(struct rte_pktmbuf_pool_private),
536                    rte_pktmbuf_pool_init, NULL,
537                    rte_pktmbuf_init, NULL,
538                    rte_socket_id(), 0);
539     if (l2fwd_pktmbuf_pool == NULL)
540         rte_exit(EXIT_FAILURE, "Cannot init mbuf pool\n");
541 
542     nb_ports = rte_eth_dev_count();  //得到物理埠的實際數量
543     if (nb_ports == 0)
544         rte_exit(EXIT_FAILURE, "No Ethernet ports - bye\n");
545 
546     if (nb_ports > RTE_MAX_ETHPORTS) //如果物理埠的數量超過限制
547         nb_ports = RTE_MAX_ETHPORTS;
548 
549     /* 重置目的物理埠的陣列  reset l2fwd_dst_ports */
550     for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++)
551         l2fwd_dst_ports[portid] = 0;//清零
552     last_port = 0;
553 
554     /*  每個lcore用在一個專用的傳送佇列上
555      * Each logical core is assigned a dedicated TX queue on each port.
556      */
557     for (portid = 0; portid < nb_ports; portid++) {//遍歷所有的物理埠
558         /* 忽略未啟用的物理埠 skip ports that are not enabled  */
559         if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
560             continue;
561 
562         if (nb_ports_in_mask % 2) { //如果是有偶數個物理埠，設為相鄰兩個物理埠對發
563             l2fwd_dst_ports[portid] = last_port; //奇數號的目的物理埠為偶數號
564             l2fwd_dst_ports[last_port] = portid; //偶數號的目的物理埠為奇數號
565         }
566         else //如果是奇數個物理埠
567             last_port = portid;
568 
569         nb_ports_in_mask++;  //更新已啟用的物理埠的總數
570 
571         rte_eth_dev_info_get(portid, &dev_info);
572     }
573     if (nb_ports_in_mask % 2) { //如果已啟用的物理埠的總數是奇數
574         printf("Notice: odd number of ports in portmask.\n");
575         l2fwd_dst_ports[last_port] = last_port;//last_port的目的物理埠還是last_port
576     }
577 
578     rx_lcore_id = 0;
579     qconf = NULL;
580 
581     /* Initialize the port/queue configuration of each logical core */
582     for (portid = 0; portid < nb_ports; portid++) { //遍歷所有的物理埠
583         /* 忽略未啟用的物理埠 skip ports that are not enabled */
584         if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
585             continue;
586 
587         /* 得到此物理埠的lcore編號  get the lcore_id for this port */
588         while (rte_lcore_is_enabled(rx_lcore_id) == 0 || //如果此lcore未啟用
589                lcore_queue_conf[rx_lcore_id].n_rx_port ==  //如果lcore上負責接收的物理埠的實際數量等於
590                l2fwd_rx_queue_per_lcore) {//每個lcore負責的接收佇列的實際數量(-q引數值)
591             rx_lcore_id++;//接收lcore的編號自增
592             if (rx_lcore_id >= RTE_MAX_LCORE) //如果接收lcore編號超過lcore最大數量
593                 rte_exit(EXIT_FAILURE, "Not enough cores\n");
594         }
595 
596         if (qconf != &lcore_queue_conf[rx_lcore_id])
597             /* Assigned a new logical core in the loop above. */
598             qconf = &lcore_queue_conf[rx_lcore_id];
599 
600         qconf->rx_port_list[qconf->n_rx_port] = portid;
601         qconf->n_rx_port++;//用於接收資料包的物理埠數量自增
602         printf("Lcore %u: RX port %u\n", rx_lcore_id, (unsigned) portid);
603     }
604 
605     nb_ports_available = nb_ports;
606 
607     /*初始化每個物理埠  Initialise each port */
608     for (portid = 0; portid < nb_ports; portid++) { //遍歷所有的物理埠
609         /* 忽略未使能的物理埠   skip ports that are not enabled */
610         if ((l2fwd_enabled_port_mask & (1 << portid)) == 0) {
611             printf("Skipping disabled port %u\n", (unsigned) portid);
612             nb_ports_available--;
613             continue;
614         }
615         /* 初始化某個物理埠 init port */
616         printf("Initializing port %u... ", (unsigned) portid);
617         fflush(stdout);
618         ret = rte_eth_dev_configure(portid, 1, 1, &port_conf); //第一步，設為1個傳送佇列和1個接收佇列
619         if (ret < 0)
620             rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%u\n",
621                   ret, (unsigned) portid);
622 
623         rte_eth_macaddr_get(portid,&l2fwd_ports_eth_addr[portid]);//獲取mac地址
624 
625         /* 在每個物理埠上建立一個接收佇列  init one RX queue */
626         fflush(stdout);
627         ret = rte_eth_rx_queue_setup(portid, 0, nb_rxd, //第二步，0代表接收佇列的編號
628                          rte_eth_dev_socket_id(portid),
629                          NULL,
630                          l2fwd_pktmbuf_pool);
631         if (ret < 0)
632             rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup:err=%d, port=%u\n",
633                   ret, (unsigned) portid);
634 
635         /* 在每個物理埠上建立一個傳送佇列    init one TX queue on each port */
636         fflush(stdout);
637         ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,//第三步，0代表傳送佇列的編號
638                 rte_eth_dev_socket_id(portid),
639                 NULL);
640         if (ret < 0)
641             rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup:err=%d, port=%u\n",
642                 ret, (unsigned) portid);
643 
644         /*啟動裝置   Start device */
645         ret = rte_eth_dev_start(portid); //第四步，啟動物理埠
646         if (ret < 0)
647             rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n",
648                   ret, (unsigned) portid);
649 
650         printf("done: \n");
651 
652         rte_eth_promiscuous_enable(portid);
653 
654         printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n",
655                 (unsigned) portid,
656                 l2fwd_ports_eth_addr[portid].addr_bytes[0],
657                 l2fwd_ports_eth_addr[portid].addr_bytes[1],
658                 l2fwd_ports_eth_addr[portid].addr_bytes[2],
659                 l2fwd_ports_eth_addr[portid].addr_bytes[3],
660                 l2fwd_ports_eth_addr[portid].addr_bytes[4],
661                 l2fwd_ports_eth_addr[portid].addr_bytes[5]);
662 
663         /*清空物理埠的統計資訊  initialize port stats */
664         memset(&port_statistics, 0, sizeof(port_statistics));
665     }
666 
667     if (!nb_ports_available) {
668         rte_exit(EXIT_FAILURE,
669             "All available ports are disabled. Please set portmask.\n");
670     }
671 
672     check_all_ports_link_status(nb_ports, l2fwd_enabled_port_mask);
673 
674     /* 在每個lcore上啟動執行緒 launch per-lcore init on every lcore */
675     rte_eal_mp_remote_launch(l2fwd_launch_one_lcore, NULL, CALL_MASTER);
676     RTE_LCORE_FOREACH_SLAVE(lcore_id) {
677         if (rte_eal_wait_lcore(lcore_id) < 0)  //等待執行緒完成工作
678             return -1;
679     }
680 
681     return 0;
682 }