From d804997da128c64d1493396da9afbc6f6cb55447 Mon Sep 17 00:00:00 2001 From: Quentin Armitage Date: Wed, 9 Dec 2015 15:29:37 +0000 Subject: Optimise closure of fds before invoking scripts Every time before a script was invoked, closeall() was called, which would spin through 1024 file descriptors closing them, even though the vast majority were not open, resulting in 1024 system calls. To avoid that, open all sockets and file descriptors (except fd 0/1/2) with the CLOEXEC flag set, so that the fds will be closed by the kernel when the script is exec'd. Signed-off-by: Quentin Armitage (cherry picked from commit 480b451a6fb98ee1c1476c3478e366666f87fe15) --- configure | 2 +- configure.in | 2 +- genhash/layer4.c | 6 +++--- keepalived/check/check_http.c | 2 +- keepalived/check/check_smtp.c | 2 +- keepalived/check/check_ssl.c | 3 +++ keepalived/check/check_tcp.c | 2 +- keepalived/core/smtp.c | 2 +- keepalived/libipvs-2.4/libipvs.c | 2 +- keepalived/libipvs-2.6/libipvs.c | 2 +- keepalived/vrrp/vrrp.c | 4 ++-- keepalived/vrrp/vrrp_arp.c | 2 +- keepalived/vrrp/vrrp_if.c | 6 +++--- keepalived/vrrp/vrrp_ndisc.c | 2 +- keepalived/vrrp/vrrp_netlink.c | 5 ++++- lib/notify.c | 11 ----------- lib/notify.h | 1 - lib/signals.c | 18 +++++++++++------- 18 files changed, 36 insertions(+), 38 deletions(-) diff --git a/configure b/configure index 9651b48..95ba428 100755 --- a/configure +++ b/configure @@ -4821,7 +4821,7 @@ cat >>confdefs.h <<_ACEOF _ACEOF -for ac_func in gettimeofday select socket strerror strtol uname +for ac_func in gettimeofday select socket strerror strtol uname pipe2 do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" diff --git a/configure.in b/configure.in index 47aa35e..fbb07a5 100644 --- a/configure.in +++ b/configure.in @@ -321,7 +321,7 @@ dnl ----[ Checks for library functions ]---- AC_PROG_GCC_TRADITIONAL AC_FUNC_MEMCMP AC_TYPE_SIGNAL -AC_CHECK_FUNCS(gettimeofday select socket strerror strtol uname) +AC_CHECK_FUNCS(gettimeofday select socket strerror strtol uname pipe2) dnl ----[ Process output target ]---- OUTPUT_TARGET="$OUTPUT_TARGET keepalived/Makefile lib/Makefile" diff --git a/genhash/layer4.c b/genhash/layer4.c index fe05fa4..3f36037 100644 --- a/genhash/layer4.c +++ b/genhash/layer4.c @@ -244,18 +244,18 @@ tcp_connect_thread(thread_t * thread) if(req->dst){ if(req->dst->ai_family == AF_INET6) { - if ((sock_obj->fd = socket(AF_INET6, SOCK_STREAM, IPPROTO_TCP)) == -1) { + if ((sock_obj->fd = socket(AF_INET6, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP)) == -1) { DBG("WEB connection fail to create socket.\n"); return 0; } } else { - if ((sock_obj->fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) == -1) { + if ((sock_obj->fd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP)) == -1) { DBG("WEB connection fail to create socket.\n"); return 0; } } } else { - if ((sock_obj->fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) == -1) { + if ((sock_obj->fd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP)) == -1) { DBG("WEB connection fail to create socket.\n"); return 0; } diff --git a/keepalived/check/check_http.c b/keepalived/check/check_http.c index 6e87252..c479ee1 100644 --- a/keepalived/check/check_http.c +++ b/keepalived/check/check_http.c @@ -870,7 +870,7 @@ http_connect_thread(thread_t * thread) } /* Create the socket */ - if ((fd = socket(co->dst.ss_family, SOCK_STREAM, IPPROTO_TCP)) == -1) { + if ((fd = socket(co->dst.ss_family, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP)) == -1) { log_message(LOG_INFO, "WEB connection fail to create socket. Rescheduling."); thread_add_timer(thread->master, http_connect_thread, checker, checker->vs->delay_loop); diff --git a/keepalived/check/check_smtp.c b/keepalived/check/check_smtp.c index 875d50d..221b87f 100644 --- a/keepalived/check/check_smtp.c +++ b/keepalived/check/check_smtp.c @@ -770,7 +770,7 @@ smtp_connect_thread(thread_t *thread) smtp_host = smtp_checker->host_ptr; /* Create the socket, failling here should be an oddity */ - if ((sd = socket(smtp_host->dst.ss_family, SOCK_STREAM, IPPROTO_TCP)) == -1) { + if ((sd = socket(smtp_host->dst.ss_family, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP)) == -1) { log_message(LOG_INFO, "SMTP_CHECK connection failed to create socket. Rescheduling."); thread_add_timer(thread->master, smtp_connect_thread, checker, checker->vs->delay_loop); diff --git a/keepalived/check/check_ssl.c b/keepalived/check/check_ssl.c index aa19011..2035a67 100644 --- a/keepalived/check/check_ssl.c +++ b/keepalived/check/check_ssl.c @@ -199,8 +199,11 @@ ssl_connect(thread_t * thread, int new_req) /* First round, create SSL context */ if (new_req) { + int bio_fd; req->ssl = SSL_new(check_data->ssl->ctx); req->bio = BIO_new_socket(thread->u.fd, BIO_NOCLOSE); + BIO_get_fd(req->bio, &bio_fd); + fcntl(bio_fd, F_SETFD, fcntl(bio_fd, F_GETFD) | FD_CLOEXEC); SSL_set_bio(req->ssl, req->bio, req->bio); } diff --git a/keepalived/check/check_tcp.c b/keepalived/check/check_tcp.c index b941ab2..c595935 100644 --- a/keepalived/check/check_tcp.c +++ b/keepalived/check/check_tcp.c @@ -131,7 +131,7 @@ tcp_connect_thread(thread_t * thread) return 0; } - if ((fd = socket(co->dst.ss_family, SOCK_STREAM, IPPROTO_TCP)) == -1) { + if ((fd = socket(co->dst.ss_family, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP)) == -1) { log_message(LOG_INFO, "TCP connect fail to create socket. Rescheduling."); thread_add_timer(thread->master, tcp_connect_thread, checker, checker->vs->delay_loop); diff --git a/keepalived/core/smtp.c b/keepalived/core/smtp.c index 4f6d685..12d487c 100644 --- a/keepalived/core/smtp.c +++ b/keepalived/core/smtp.c @@ -583,7 +583,7 @@ smtp_connect(smtp_t * smtp) { enum connect_result status; - if ((smtp->fd = socket(global_data->smtp_server.ss_family, SOCK_STREAM, IPPROTO_TCP)) == -1) { + if ((smtp->fd = socket(global_data->smtp_server.ss_family, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_TCP)) == -1) { DBG("SMTP connect fail to create socket."); free_smtp_all(smtp); return; diff --git a/keepalived/libipvs-2.4/libipvs.c b/keepalived/libipvs-2.4/libipvs.c index be0329e..ff95e8c 100644 --- a/keepalived/libipvs-2.4/libipvs.c +++ b/keepalived/libipvs-2.4/libipvs.c @@ -35,7 +35,7 @@ int ipvs_init(void) socklen_t len; len = sizeof(ipvs_info); - if ((sockfd = socket(AF_INET, SOCK_RAW, IPPROTO_RAW)) == -1) + if ((sockfd = socket(AF_INET, SOCK_RAW | SOCK_CLOEXEC, IPPROTO_RAW)) == -1) return -1; ipvs_cmd = GET_CMD(IP_VS_SO_GET_INFO); diff --git a/keepalived/libipvs-2.6/libipvs.c b/keepalived/libipvs-2.6/libipvs.c index e2df4c8..86d6d73 100644 --- a/keepalived/libipvs-2.6/libipvs.c +++ b/keepalived/libipvs-2.6/libipvs.c @@ -168,7 +168,7 @@ int ipvs_init(void) #endif len = sizeof(ipvs_info); - if ((sockfd = socket(AF_INET, SOCK_RAW, IPPROTO_RAW)) == -1) + if ((sockfd = socket(AF_INET, SOCK_RAW | SOCK_CLOEXEC, IPPROTO_RAW)) == -1) return -1; if (getsockopt(sockfd, IPPROTO_IP, IP_VS_SO_GET_INFO, diff --git a/keepalived/vrrp/vrrp.c b/keepalived/vrrp/vrrp.c index 22ae856..431400f 100644 --- a/keepalived/vrrp/vrrp.c +++ b/keepalived/vrrp/vrrp.c @@ -1445,7 +1445,7 @@ open_vrrp_send_socket(sa_family_t family, int proto, int idx, int unicast) ifp = if_get_by_ifindex(idx); /* Create and init socket descriptor */ - fd = socket(family, SOCK_RAW, proto); + fd = socket(family, SOCK_RAW | SOCK_CLOEXEC, proto); if (fd < 0) { log_message(LOG_INFO, "cant open raw socket. errno=%d", errno); return -1; @@ -1488,7 +1488,7 @@ open_vrrp_socket(sa_family_t family, int proto, int idx, ifp = if_get_by_ifindex(idx); /* open the socket */ - fd = socket(family, SOCK_RAW, proto); + fd = socket(family, SOCK_RAW | SOCK_CLOEXEC, proto); if (fd < 0) { int err = errno; log_message(LOG_INFO, "cant open raw socket. errno=%d", err); diff --git a/keepalived/vrrp/vrrp_arp.c b/keepalived/vrrp/vrrp_arp.c index e53b9d7..58116a9 100644 --- a/keepalived/vrrp/vrrp_arp.c +++ b/keepalived/vrrp/vrrp_arp.c @@ -98,7 +98,7 @@ void gratuitous_arp_init(void) garp_buffer = (char *)MALLOC(sizeof(arphdr_t) + ETHER_HDR_LEN); /* Create the socket descriptor */ - garp_fd = socket(PF_PACKET, SOCK_RAW, htons(ETH_P_RARP)); + garp_fd = socket(PF_PACKET, SOCK_RAW | SOCK_CLOEXEC, htons(ETH_P_RARP)); if (garp_fd > 0) log_message(LOG_INFO, "Registering gratuitous ARP shared channel"); diff --git a/keepalived/vrrp/vrrp_if.c b/keepalived/vrrp/vrrp_if.c index 38fecf1..4d914eb 100644 --- a/keepalived/vrrp/vrrp_if.c +++ b/keepalived/vrrp/vrrp_if.c @@ -201,7 +201,7 @@ if_mii_probe(const char *ifname) { uint16_t *data = (uint16_t *) (&ifr.ifr_data); int phy_id; - int fd = socket(AF_INET, SOCK_DGRAM, 0); + int fd = socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0); int status = 0; if (fd < 0) @@ -250,7 +250,7 @@ if_ethtool_status(const int fd) int if_ethtool_probe(const char *ifname) { - int fd = socket(AF_INET, SOCK_DGRAM, 0); + int fd = socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0); int status = 0; if (fd < 0) @@ -266,7 +266,7 @@ if_ethtool_probe(const char *ifname) void if_ioctl_flags(interface_t * ifp) { - int fd = socket(AF_INET, SOCK_DGRAM, 0); + int fd = socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0); if (fd < 0) return; diff --git a/keepalived/vrrp/vrrp_ndisc.c b/keepalived/vrrp/vrrp_ndisc.c index 8ce1e2e..07daf06 100644 --- a/keepalived/vrrp/vrrp_ndisc.c +++ b/keepalived/vrrp/vrrp_ndisc.c @@ -188,7 +188,7 @@ ndisc_init(void) sizeof(struct ndhdr) + sizeof(struct nd_opt_hdr) + ETH_ALEN); /* Create the socket descriptor */ - ndisc_fd = socket(PF_PACKET, SOCK_RAW, htons(ETH_P_IPV6)); + ndisc_fd = socket(PF_PACKET, SOCK_RAW | SOCK_CLOEXEC, htons(ETH_P_IPV6)); } void diff --git a/keepalived/vrrp/vrrp_netlink.c b/keepalived/vrrp/vrrp_netlink.c index c2b2702..bb841aa 100644 --- a/keepalived/vrrp/vrrp_netlink.c +++ b/keepalived/vrrp/vrrp_netlink.c @@ -105,11 +105,14 @@ netlink_socket(nl_handle_t *nl, int flags, int group, ...) nl->nl_pid = nl_socket_get_local_port(nl->sk); nl->fd = nl_socket_get_fd(nl->sk); + + /* Set CLOEXEC */ + fcntl(nl->fd, F_SETFD, fcntl(nl->fd, F_GETFD) | FD_CLOEXEC); #else socklen_t addr_len; struct sockaddr_nl snl; - nl->fd = socket(AF_NETLINK, SOCK_RAW | flags, NETLINK_ROUTE); + nl->fd = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC | flags, NETLINK_ROUTE); if (nl->fd < 0) { log_message(LOG_INFO, "Netlink: Cannot open netlink socket : (%s)", strerror(errno)); diff --git a/lib/notify.c b/lib/notify.c index 5e375ab..f060877 100644 --- a/lib/notify.c +++ b/lib/notify.c @@ -53,20 +53,9 @@ system_call(const char *cmdline) return retval; } -/* Close all FDs >= a specified value */ -void -closeall(int fd) -{ - int fdlimit = sysconf(_SC_OPEN_MAX); - while (fd < fdlimit) - close(fd++); -} - static void script_setup(void) { - closeall(0); - signal_handler_script(); set_std_fd(false); diff --git a/lib/notify.h b/lib/notify.h index 6dbc561..6067d95 100644 --- a/lib/notify.h +++ b/lib/notify.h @@ -27,7 +27,6 @@ /* system includes */ extern int system_call_script(thread_master_t *m, int (*func) (thread_t *), void * arg, long timer, const char* script); -extern void closeall(int fd); extern int notify_exec(char *cmd); #endif diff --git a/lib/signals.c b/lib/signals.c index 0e46fd9..6082e42 100644 --- a/lib/signals.c +++ b/lib/signals.c @@ -171,12 +171,21 @@ signal_handler_init(void) sigset_t sset; int sig; struct sigaction act, oact; - int n = pipe(signal_pipe); - assert(!n); + int n; + +#ifdef HAVE_PIPE2 + n = pipe2(signal_pipe, O_CLOEXEC | O_NONBLOCK); +#else + n = pipe(signal_pipe); fcntl(signal_pipe[0], F_SETFL, O_NONBLOCK | fcntl(signal_pipe[0], F_GETFL)); fcntl(signal_pipe[1], F_SETFL, O_NONBLOCK | fcntl(signal_pipe[1], F_GETFL)); + fcntl(signal_pipe[0], F_SETFD, FD_CLOEXEC | fcntl(signal_pipe[0], F_GETFD)); + fcntl(signal_pipe[1], F_SETFD, FD_CLOEXEC | fcntl(signal_pipe[1], F_GETFD)); +#endif + assert(!n); + signal_SIGHUP_handler = NULL; signal_SIGINT_handler = NULL; signal_SIGTERM_handler = NULL; @@ -270,11 +279,6 @@ signal_handler_script(void) else if (sigismember(&dfl_sig, sig)) sigaction(sig, &dfl, NULL); } - - close(signal_pipe[1]); - close(signal_pipe[0]); - signal_pipe[1] = -1; - signal_pipe[0] = -1; } int -- 1.7.12.1