使用netfilter_queue改包筆記
阿新 • • 發佈：2017-11-26
sad 回調 編譯 val fun oid %d handle quit
系統：centos 7
準備：安裝libnetfilter_queue模塊，可以yum安裝，也可以網上下載rpm包安裝
簡介：使用iptables在NAT表上創建DNAT與SNAT規則，對數據包進行轉發；在MANGLE表上的FORWARD鏈上創建NF_QUEUE規則對數據進行勾取並修改；（iptables只有mangle表可以修改數據）
示例規則：
//把到本機 50.24 8889端口的數據包，nat到50.4的8889端口 iptables -t nat -A PREROUTING -p udp -d 192.168.50.24 --dport 8889 -j DNAT --to 192.168.50.4 iptables -t nat -A POSTROUTING -p udp -d 192.168.50.4 --dport 8889 -j SNAT --to 192.168.50.24 //把目的地址50.4，目的端口8889的數據包，入隊列 1 iptables -t mangle -A FORWARD -d 192.168.50.4 -p tcp --dport 8889 -j NFQUEUE --queue-num 1
示例代碼：
主線程DoListenIptablesThread負責對QUEUE隊列數據的讀取，讀取到的數據通過回調PacketHandler方法解析處理，傳入參數為 queue的 ID號
static void *DoListenIptablesThread(void *pData) { struct nfq_handle *h; struct nfq_q_handle *qh; struct nfnl_handle *nh; int fd; int rv; int i; pthread_t RecvPth[PthNUM]; char buf[QUEUE_BUFSIZE]; TCLEANFUNCT struTmp; int nTmpError = -1; int nNum = *(int*)pData;
free(pData);
pthread_detach(pthread_self()); memset(&struTmp, 0, sizeof(struTmp)); zlog_debug(cat,"opening library handle, nNum[%d]", nNum); h = nfq_open(); if (!h) { nTmpError = errno; zlog_debug(cat,"error during nfq_open(), nNum[%d]", nNum); zlog_debug(cat,"nfq_open() errno[%d][%s]", nTmpError, strerror(nTmpError)); pthread_exit(0); } zlog_debug(cat,"unbinding existing nf_queue handler for AF_INET (if any), nNum[%d]", nNum); if (nfq_unbind_pf(h, AF_INET) < 0) { nTmpError = errno; zlog_debug(cat,"error during nfq_unbind_pf(), nNum[%d]", nNum); zlog_debug(cat,"nfq_unbind_pf() errno[%d][%s]", nTmpError, strerror(nTmpError)); nfq_close(h); pthread_exit(0); } zlog_debug(cat,"binding nfnetlink_queue as nf_queue handler for AF_INET, nNum[%d]", nNum); if (nfq_bind_pf(h, AF_INET) < 0) { nTmpError = errno; zlog_debug(cat,"error during nfq_bind_pf(), nNum[%d]", nNum); zlog_debug(cat,"nfq_bind_pf() errno[%d][%s]", nTmpError, strerror(nTmpError)); nfq_close(h); pthread_exit(0); } zlog_debug(cat,"binding this socket to queue [%d]", nNum); qh = nfq_create_queue(h, nNum, &PacketHandler, &nNum); if (!qh) { nTmpError = errno; zlog_debug(cat,"error during nfq_create_queue(), nNum[%d]", nNum); zlog_debug(cat,"nfq_create_queue() errno[%d][%s]", nTmpError, strerror(nTmpError)); nfq_close(h); pthread_exit(0); } zlog_debug(cat,"setting copy_packet mode, nNum[%d]", nNum); if (nfq_set_mode(qh, NFQNL_COPY_PACKET, 0xffff) < 0) { nTmpError = errno; zlog_debug(cat,"can‘t set packet_copy mode, nNum[%d]", nNum); zlog_debug(cat,"nfq_set_mode() errno[%d][%s]", nTmpError, strerror(nTmpError)); nfq_destroy_queue(qh); nfq_close(h); pthread_exit(0); } nh = nfq_nfnlh(h); fd = nfnl_fd(nh); struTmp.qh = qh; struTmp.h = h; for(i = 0;i<PthNUM;i++){ pthread_create(&RecvPth[i], NULL, DoRecvPacketThread,(void*)&struTmp); struTmp.RecvPth[i] = RecvPth[i]; } pthread_cleanup_push(FreePorcessResource, (void*)&struTmp); zlog_debug(cat,"Waitting for message ..., nNum[%d]", nNum); while ((rv = recv(fd, buf, sizeof(buf), 0)) && rv >= 0) { // 開始處理數據 //zlog_debug(cat,"-- New packet received -- rv[%d]", rv); nfq_handle_packet(h, buf, rv);
memset(buf,0x00,sizeof(buf)); } if (rv < 0) { if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { zlog_debug(cat, "error: [%s], wait for next event.", strerror(errno)); } else { // recv error, free conncetion. zlog_error(cat,"recv error: [%s]",strerror(errno)); } } pthread_cleanup_pop(0); zlog_error(cat,"-- New packet received -- rv[%d] fd = [%d]", rv,fd); zlog_debug(cat,"Exit DoNetFilter"); }
static int PacketHandler(struct nfq_q_handle *qh, struct nfgenmsg *nfmsg,struct nfq_data *nfa, void *data) { int id = 0; struct nfqnl_msg_packet_hdr *ph; u_int32_t mark,ifi; struct iphdr *iph; int iphdr_size; int ret;char *nf_packet; unsigned int nAppProto = -1; int nReturnValue = 0; char szHost[30] = {0}; ph = nfq_get_msg_packet_hdr(nfa); if (ph) { id = ntohl(ph->packet_id); } mark = nfq_get_nfmark(nfa); if (mark) { // DEBUG_LOG("mark=%u ", mark); } ifi = nfq_get_indev(nfa); if (ifi) { // DEBUG_LOG("indev=%u ", ifi); } ifi = nfq_get_outdev(nfa); if (ifi) { // DEBUG_LOG("outdev=%u ", ifi); } ret = nfq_get_payload(nfa, (unsigned char**)&nf_packet); if ((ret >= 0)) { //DEBUG_LOG("payload_len=%d bytes", ret); //fputc(‘\n‘, stdout); } // parse the packet headers iph = ((struct iphdr *) nf_packet); iphdr_size = iph->ihl << 2; if (iph->protocol == TCP_PRO) { struct tcphdr *tcp; int tcphdr_size; int clen; tcp = ((struct tcphdr *) (nf_packet + (iph->ihl << 2))); tcphdr_size = (tcp->doff << 2); clen = ret - iphdr_size - tcphdr_size; if(clen > 0) {
//在此處修改數據包，修改數據包後執行下面兩行代碼，重新對數據進行校驗，然後通知內核放行修改後的數據包 //set_tcp_checksum1(iph);
//return nfq_set_verdict(qh, id, NF_ACCEPT,(u_int32_t)ret, nf_packet); } } // if protocol is udp if(iph->protocol == UDP_PRO) { int clen; struct udphdr *udp; udp = ((struct udphdr *) (nf_packet + (iph->ihl << 2))); clen = ret - iphdr_size - UDP_HEADER_LEN; if(clen > 0) { char* c; PACKETINFO packinfo; memset(&packinfo,0x00, sizeof(struct PACKETINFO)); c = nf_packet + iphdr_size + UDP_HEADER_LEN;
//在此處修改數據包，修改數據包後執行下面兩行代碼，重新對數據進行校驗，然後通知內核放行修改後的數據包 //set_udp_checksum1(iph); //return nfq_set_verdict(qh, id, NF_ACCEPT,(u_int32_t)ret, nf_packet); } } return nfq_set_verdict(qh, id, NF_ACCEPT,0, NULL); }
線程退出時資源釋放代碼：
主線程DoListenIptablesThread中recv的行為為阻塞，所以強制通過其他方式強制退出時，無法有效關閉並釋放資源，通過FreePorcessResource對其資源進行關閉回收，並殺掉其開辟的線程；
void FreePorcessResource(void *pData) { TCLEANFUNCT *pTmp = NULL; int i; int kill_rc; pTmp = (TCLEANFUNCT *)pData; for(i = 0;i<PthNUM;i++){ if(!pTmp->RecvPth[i]) continue; kill_rc = pthread_kill(pTmp->RecvPth[i], 0); if (kill_rc == ESRCH) { zlog_debug(cat,"the specified thread did not exists or already quit --- "); } else if (kill_rc == EINVAL) { zlog_debug(cat,"signal is invalid --- "); } else { zlog_debug(cat,"the specified thread is alive --- "); // 殺死該線程 pthread_cancel(pTmp->RecvPth[i]); //pthread_join(m->second, NULL); usleep(50*1000); // 檢測該線程是否存在 kill_rc = pthread_kill(pTmp->RecvPth[i], 0); if (kill_rc == ESRCH) { zlog_debug(cat,"the specified thread did not exists or already quit +++ "); } else if (kill_rc == EINVAL) { zlog_debug(cat,"signal is invalid +++ "); } else { zlog_debug(cat,"signal is alive +++ "); } } } nfq_destroy_queue(pTmp->qh); nfq_close(pTmp->h); zlog_debug(cat,"closing pthread handle\n"); }
主線程DoListenIptablesThread創建的數據讀取線程：(多核設備時，內核會通過多核接收數據，單線程recv數據時，系統接收緩存區會由於應用層recv過慢造成緩存區沒有足夠的空間，所以該處需要多線程recv處理)
static void *DoRecvPacketThread(void *pData){ TCLEANFUNCT *pTmp = NULL; int rv; int fd; char buf[QUEUE_BUFSIZE]; struct nfnl_handle *nh; pthread_detach(pthread_self()); pTmp = (TCLEANFUNCT *)pData; nh = nfq_nfnlh(pTmp->h); fd = nfnl_fd(nh); while ((rv = recv(fd, buf, sizeof(buf), 0)) && rv >= 0) { // 開始處理數據 //zlog_debug(cat,"-- New packet received -- rv[%d]", rv); nfq_handle_packet(pTmp->h, buf, rv);
memset(buf,0x00,sizeof(buf)); } if (rv < 0) { if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { zlog_debug(cat, "error: [%s], wait for next event.", strerror(errno)); } else { // recv error, free conncetion. zlog_error(cat,"recv error: [%s]",strerror(errno)); } } zlog_error(cat,"-- New packet received -- rv[%d] fd = [%d]", rv,fd); }
TCP與UDP數據修改後重新校驗實現：
static u_int16_t checksum(u_int32_t init, u_int8_t *addr, size_t count){ /* Compute Internet Checksum for "count" bytes * beginning at location "addr". */ u_int32_t sum = init; while( count > 1 ) { /* This is the inner loop */ sum += ntohs(* (u_int16_t*) addr); addr += 2; count -= 2; } /* Add left-over byte, if any */ if( count > 0 ) sum += ntohs(* (u_int8_t*) addr); /* Fold 32-bit sum to 16 bits */ while (sum>>16) sum = (sum & 0xffff) + (sum >> 16); return (u_int16_t)~sum; } static u_int16_t tcp_checksum2(struct iphdr* iphdrp, struct tcphdr* tcphdrp){ size_t tcplen = ntohs(iphdrp->tot_len) - (iphdrp->ihl<<2); u_int32_t cksum = 0; cksum += ntohs((iphdrp->saddr >> 16) & 0x0000ffff); cksum += ntohs(iphdrp->saddr & 0x0000ffff); cksum += ntohs((iphdrp->daddr >> 16) & 0x0000ffff); cksum += ntohs(iphdrp->daddr & 0x0000ffff); cksum += iphdrp->protocol & 0x00ff; cksum += tcplen; return checksum(cksum, (u_int8_t*)tcphdrp, tcplen); } static u_int16_t tcp_checksum1(struct iphdr* iphdrp){ struct tcphdr *tcphdrp = (struct tcphdr*)((u_int8_t*)iphdrp + (iphdrp->ihl<<2)); return tcp_checksum2(iphdrp, tcphdrp); } static void set_tcp_checksum2(struct iphdr* iphdrp, struct tcphdr* tcphdrp){ tcphdrp->check = 0; tcphdrp->check = htons(tcp_checksum2(iphdrp, tcphdrp)); } static void set_tcp_checksum1(struct iphdr* iphdrp){ struct tcphdr *tcphdrp = (struct tcphdr*)((u_int8_t*)iphdrp + (iphdrp->ihl<<2)); set_tcp_checksum2(iphdrp, tcphdrp); } static u_int16_t udp_checksum2(struct iphdr* iphdrp, struct udphdr* udphdrp){ size_t udplen = ntohs(iphdrp->tot_len) - (iphdrp->ihl<<2); u_int32_t cksum = 0; cksum += ntohs((iphdrp->saddr >> 16) & 0x0000ffff); cksum += ntohs(iphdrp->saddr & 0x0000ffff); cksum += ntohs((iphdrp->daddr >> 16) & 0x0000ffff); cksum += ntohs(iphdrp->daddr & 0x0000ffff); cksum += iphdrp->protocol & 0x00ff; cksum += udplen; return checksum(cksum, (u_int8_t*)udphdrp, udplen); } static u_int16_t udp_checksum1(struct iphdr* iphdrp){ struct udphdr *udphdrp = (struct udphdr*)((u_int8_t*)iphdrp + (iphdrp->ihl<<2)); return udp_checksum2(iphdrp, udphdrp); } static void set_udp_checksum2(struct iphdr* iphdrp, struct udphdr* udphdrp){ udphdrp->check = 0; udphdrp->check = htons(udp_checksum2(iphdrp, udphdrp)); } static void set_udp_checksum1(struct iphdr* iphdrp){ struct udphdr *udphdrp = (struct udphdr*)((u_int8_t*)iphdrp + (iphdrp->ihl<<2)); set_udp_checksum2(iphdrp, udphdrp); }
程序內部宏定義整理：
#define MAC_LEN 12 #define UDP_PRO 17 #define TCP_PRO 6 #define VXLAN_HEADER_LEN 8 #define UDP_HEADER_LEN 8 #define TCP_HEADER_NO_OPERATION_LEN 20 #define QUEUE_BUFSIZE 8192 #define PthNUM 10 typedef struct __CleanFunct { struct nfq_q_handle *qh; struct nfq_handle *h; pthread_t RecvPth[PthNUM]; }TCLEANFUNCT;
程序需要頭文件：
#include <stdio.h> #include <stdlib.h> #include <unistd.h> #include <string.h> #include <linux/ip.h> #include <linux/tcp.h> #include <linux/udp.h> #include <netinet/in.h> #include <linux/types.h> #include <pthread.h> #include <zlog.h> #include <assert.h> #include <linux/netfilter.h> #include <libnetfilter_queue/libnetfilter_queue.h>
代碼編譯需要鏈接內容：
-lpthread -lnfnetlink -lnetfilter_queue
整理不易，轉載請註明出處；
使用netfilter_queue改包筆記