/* * Copyright (c) 2014, Peter Haag * Copyright (c) 2013, Peter Haag * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * * Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * Neither the name of the author nor the names of its contributors may be * used to endorse or promote products derived from this software without * specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * $Author$ * * $Id$ * * $LastChangedRevision$ * */ /* $Id: pcapd.c 2778 2012-03-19 09:23:26Z roethlis $ */ #include "config.h" #ifdef HAVE_FEATURES_H #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef HAVE_STDINT_H #include #endif #include #include #include #include #include #include #include #include #ifdef HAVE_NET_BPF_H # include #else #ifdef HAVE_PCAP_BPF_H # include #else # error missing bpf header #endif #endif #include #include "util.h" #include "nffile.h" #include "nfx.h" #include "nf_common.h" #include "nfnet.h" #include "flist.h" #include "nfstatfile.h" #include "bookkeeper.h" #include "nfxstat.h" #include "collector.h" #include "exporter.h" #ifdef HAVE_FTS_H # include #else # include "fts_compat.h" #define fts_children fts_children_compat #define fts_close fts_close_compat #define fts_open fts_open_compat #define fts_read fts_read_compat #define fts_set fts_set_compat #endif #include "expire.h" #include "flowtree.h" #include "netflow_pcap.h" #include "pcaproc.h" #define TIME_WINDOW 300 #define SNAPLEN 200 #define PROMISC 1 #define TIMEOUT 500 #define FILTER "" #define DEFAULT_DIR "/var/tmp" #ifndef DLT_LINUX_SLL #define DLT_LINUX_SLL 113 #endif #ifndef DEVEL # define dbg_printf(...) /* printf(__VA_ARGS__) */ #else # define dbg_printf(...) printf(__VA_ARGS__) #endif int verbose = 0; /* * global static var: used by interrupt routine */ #define PCAP_DUMPFILE "pcap.current" static const char *nfdump_version = VERSION; static int launcher_alive, periodic_trigger, launcher_pid; static pthread_mutex_t m_done = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t terminate = PTHREAD_COND_INITIALIZER; static pthread_key_t buffer_key; uint32_t linktype; uint32_t linkoffset; // Common thread info struct typedef struct thread_info_s { pthread_t tid; int done; int exit; } thread_info_t; typedef struct p_pcap_flush_thread_args_s { // common thread info struct pthread_t tid; int done; int exit; // the parent pthread_t parent; // arguments int subdir_index; char *pcap_datadir; pcap_dev_t *pcap_dev; pcapfile_t *pcapfile; } p_pcap_flush_thread_args_t; typedef struct p_packet_thread_args_s { // common thread info struct pthread_t tid; int done; int exit; // the parent pthread_t parent; // arguments NodeList_t *NodeList; // push new nodes into this list pcap_dev_t *pcap_dev; time_t t_win; int subdir_index; char *pcap_datadir; int live; } p_packet_thread_args_t; typedef struct p_flow_thread_args_s { // common thread info struct pthread_t tid; int done; int exit; // the parent pthread_t parent; // arguments NodeList_t *NodeList; // pop new nodes from this list FlowSource_t *fs; time_t t_win; int subdir_index; int compress; } p_flow_thread_args_t; /* * Function prototypes */ static void usage(char *name); static void daemonize(void); static void Interrupt_handler(int sig); static void SetPriv(char *userid, char *groupid ); static pcap_dev_t *setup_pcap_live(char *device, char *filter, int snaplen); static pcap_dev_t *setup_pcap_Ffile(FILE *fp, char *filter, int snaplen); static pcap_dev_t *setup_pcap_file(char *pcap_file, char *filter, int snaplen); static void WaitDone(void); static void SignalThreadTerminate(thread_info_t *thread_info, pthread_cond_t *thread_cond ); static void *p_pcap_flush_thread(void *thread_data); static void *p_flow_thread(void *thread_data); static void *p_packet_thread(void *thread_data); /* * Functions */ static void usage(char *name) { printf("usage %s [options] [\"pcap filter\"]\n" "-h\t\tthis text you see right here\n" "-u userid\tChange user to username\n" "-g groupid\tChange group to groupname\n" "-i device\tspecify a device\n" "-r pcapfile\tspecify a file to read from\n" "-B cache buckets\tset the number of cache buckets. (default 1048576)\n" "-s snaplen\tset the snapshot length\n" "-l flowdir \tset the flow output directory. (no default) \n" "-l pcapdir \tset the pcapdir directory. (optional) \n" "-S subdir\tSub directory format. see nfcapd(1) for format\n" "-I Ident\tset the ident string for stat file. (default 'none')\n" "-P pidfile\tset the PID file\n" "-t time frame\tset the time window to rotate pcap/nfcapd file\n" "-z\t\tCompress flows in output file.\n" "-E\t\tPrint extended format of netflow data. for debugging purpose only.\n" "-T\t\tInclude extension tags in records.\n" "-D\t\tdetach from terminal (daemonize)\n" , name); } // End of usage static void Interrupt_handler(int sig) { pthread_t tid = pthread_self(); thread_info_t *thread_info; LogInfo("[%lu] Signal handler: %i", (long unsigned)tid, sig); thread_info = (thread_info_t *)pthread_getspecific(buffer_key); if ( !thread_info ) { LogError("[%lu] Interrupt_handler() failed to get thread specific data block\n", (long unsigned)tid); } else { if ( thread_info->tid != tid ) { LogError("[%lu] Interrupt_handler() missmatch tid in thread_info\n", (long unsigned)tid); } else { thread_info->done = 1; } } } // End of signal_handler static void daemonize(void) { int fd; switch (fork()) { case 0: // child break; case -1: // error fprintf(stderr, "fork() error: %s\n", strerror(errno)); exit(0); break; default: // parent _exit(0); } if (setsid() < 0) { fprintf(stderr, "setsid() error: %s\n", strerror(errno)); exit(0); } // Double fork switch (fork()) { case 0: // child break; case -1: // error fprintf(stderr, "fork() error: %s\n", strerror(errno)); exit(0); break; default: // parent _exit(0); } fd = open("/dev/null", O_RDONLY); if (fd != 0) { dup2(fd, 0); close(fd); } fd = open("/dev/null", O_WRONLY); if (fd != 1) { dup2(fd, 1); close(fd); } fd = open("/dev/null", O_WRONLY); if (fd != 2) { dup2(fd, 2); close(fd); } } // End of daemonize static void SetPriv(char *userid, char *groupid ) { struct passwd *pw_entry; struct group *gr_entry; uid_t myuid, newuid, newgid; int err; if ( userid == 0 && groupid == 0 ) return; newuid = newgid = 0; myuid = getuid(); if ( myuid != 0 ) { LogError("Only root wants to change uid/gid"); fprintf(stderr, "ERROR: Only root wants to change uid/gid\n"); exit(255); } if ( userid ) { pw_entry = getpwnam(userid); newuid = pw_entry ? pw_entry->pw_uid : atol(userid); if ( newuid == 0 ) { fprintf (stderr,"Invalid user '%s'\n", userid); exit(255); } } if ( groupid ) { gr_entry = getgrnam(groupid); newgid = gr_entry ? gr_entry->gr_gid : atol(groupid); if ( newgid == 0 ) { fprintf (stderr,"Invalid group '%s'\n", groupid); exit(255); } err = setgid(newgid); if ( err ) { LogError("Can't set group id %ld for group '%s': %s", (long)newgid, groupid, strerror(errno)); fprintf (stderr,"Can't set group id %ld for group '%s': %s\n", (long)newgid, groupid, strerror(errno)); exit(255); } } if ( newuid ) { err = setuid(newuid); if ( err ) { LogError("Can't set user id %ld for user '%s': %s", (long)newuid, userid, strerror(errno)); fprintf (stderr,"Can't set user id %ld for user '%s': %s\n", (long)newuid, userid, strerror(errno)); exit(255); } } } // End of SetPriv static pcap_dev_t *setup_pcap_live(char *device, char *filter, int snaplen) { pcap_t *handle; pcap_dev_t *pcap_dev; char errbuf[PCAP_ERRBUF_SIZE]; bpf_u_int32 mask; /* Our netmask */ bpf_u_int32 net; /* Our IP */ struct bpf_program filter_code; uint32_t linkoffset, linktype; dbg_printf("Enter function: %s\n", __FUNCTION__); if (device == NULL) { device = pcap_lookupdev(errbuf); if (device == NULL) { LogError("Couldn't find default device: %s", errbuf); return NULL; } } /* Find the properties for the device */ if (pcap_lookupnet(device, &net, &mask, errbuf) == -1) { LogError("Couldn't get netmask for device %s: %s", device, errbuf); net = 0; mask = 0; } /* * Open the packet capturing device with the following values: * * SNAPLEN: User defined or 200 bytes * PROMISC: on * The interface needs to be in promiscuous mode to capture all * network traffic on the localnet. * TIMEOUT: 500ms * A 500 ms timeout is probably fine for most networks. For * architectures that support it, you might want tune this value * depending on how much traffic you're seeing on the network. */ handle = pcap_open_live(device, snaplen, PROMISC, TIMEOUT, errbuf); if (handle == NULL) { LogError("Couldn't open device %s: %s", device, errbuf); return NULL; } // XXX // int pcap_set_buffer_size(pcap_t *p, int buffer_size); if ( filter ) { /* Compile and apply the filter */ if (pcap_compile(handle, &filter_code, filter, 0, net) == -1) { LogError("Couldn't parse filter %s: %s", filter, pcap_geterr(handle)); return NULL; } if (pcap_setfilter(handle, &filter_code) == -1) { LogError("Couldn't install filter %s: %s", filter, pcap_geterr(handle)); return NULL; } } pcap_dev = (pcap_dev_t *)calloc(1, sizeof(pcap_dev_t)); if ( !pcap_dev ) { LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); return NULL; } linkoffset = 0; linktype = pcap_datalink(handle); switch ( linktype ) { case DLT_RAW: linkoffset = 0; break; case DLT_PPP: linkoffset = 2; break; case DLT_NULL: linkoffset = 4; break; case DLT_LOOP: linkoffset = 14; break; case DLT_EN10MB: linkoffset = 14; break; case DLT_LINUX_SLL: linkoffset = 16; break; case DLT_IEEE802_11: linkoffset = 22; break; default: LogError("Unsupported data link type %i", linktype); return NULL; } pcap_dev->handle = handle; pcap_dev->snaplen = snaplen; pcap_dev->linkoffset = linkoffset; pcap_dev->linktype = linktype; return pcap_dev; } // End of setup_pcap_live static pcap_dev_t *setup_pcap_file(char *pcap_file, char *filter, int snaplen) { FILE *fp; fp = fopen(pcap_file, "rb"); if ( !fp ) { LogError("Couldn't open file: %s: %s", pcap_file, strerror(errno)); return NULL; } return setup_pcap_Ffile(fp, filter, snaplen); } // End of setup_pcap_file static pcap_dev_t *setup_pcap_Ffile(FILE *fp, char *filter, int snaplen) { pcap_t *handle; pcap_dev_t *pcap_dev; char errbuf[PCAP_ERRBUF_SIZE]; uint32_t linkoffset, linktype; dbg_printf("Enter function: %s\n", __FUNCTION__); if ( !fp ) return NULL; handle = pcap_fopen_offline(fp, errbuf); if (handle == NULL) { LogError("Couldn't attach FILE handle %s", errbuf); return NULL; } if ( filter ) { struct bpf_program filter_code; bpf_u_int32 netmask = 0; // Compile and apply the filter if (pcap_compile(handle, &filter_code, filter, 0, netmask) == -1) { LogError("Couldn't parse filter %s: %s", filter, pcap_geterr(handle)); return NULL; } if (pcap_setfilter(handle, &filter_code) == -1) { LogError("Couldn't install filter %s: %s", filter, pcap_geterr(handle)); return NULL; } } linkoffset = 0; linktype = pcap_datalink(handle); switch ( linktype ) { case DLT_RAW: linkoffset = 0; break; case DLT_PPP: linkoffset = 2; break; case DLT_NULL: linkoffset = 4; break; case DLT_LOOP: linkoffset = 14; break; case DLT_EN10MB: linkoffset = 14; break; case DLT_LINUX_SLL: linkoffset = 16; break; case DLT_IEEE802_11: linkoffset = 22; break; default: LogError("Unsupported data link type %i", linktype); return NULL; } pcap_dev = (pcap_dev_t *)calloc(1, sizeof(pcap_dev_t)); if ( !pcap_dev ) { LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); return NULL; } pcap_dev->handle = handle; pcap_dev->snaplen = snaplen; pcap_dev->linkoffset = linkoffset; pcap_dev->linktype = linktype; return pcap_dev; } // End of setup_pcap_file static void SignalThreadTerminate(thread_info_t *thread_info, pthread_cond_t *thread_cond ) { if ( !thread_info->done ) { dbg_printf("Signal thread[%lu] to terminate\n", (long unsigned)thread_info->tid); if ( pthread_kill(thread_info->tid, SIGUSR2) != 0 ) { dbg_printf("Failed to signal thread[%lu]\n", (long unsigned)thread_info->tid); } // in case of a condition - signal condition if ( thread_cond ) pthread_cond_signal(thread_cond); } else { dbg_printf("thread[%lu] gone already\n", (long unsigned)thread_info->tid); } if( pthread_join(thread_info->tid, NULL) == 0 ) { dbg_printf("thread %lu joined\n", (long unsigned)thread_info->tid); } else { dbg_printf("thread %lu no join\n", (long unsigned)thread_info->tid); } LogInfo("Exit status thread[%lu]: %i", thread_info->tid, thread_info->exit); } // End of SignalThreadEnd __attribute__((noreturn)) static void *p_flow_thread(void *thread_data) { // argument dispatching p_flow_thread_args_t *args = (p_flow_thread_args_t *)thread_data; time_t t_win = args->t_win; int subdir_index = args->subdir_index; int compress = args->compress; FlowSource_t *fs = args->fs; // locals time_t t_start, t_clock, t_udp_flush; int err, done; done = 0; args->done = 0; args->exit = 0; err = pthread_setspecific( buffer_key, (void *)args ); if ( err ) { LogError("[%lu] pthread_setspecific() error in %s line %d: %s\n", (long unsigned)args->tid, __FILE__, __LINE__, strerror(errno) ); args->done = 1; args->exit = 255; pthread_kill(args->parent, SIGUSR1); pthread_exit((void *)args); } if ( !Init_pcap2nf() ) { args->done = 1; args->exit = 255; pthread_kill(args->parent, SIGUSR1); pthread_exit((void *)args); } // prepare file fs->nffile = OpenNewFile(fs->current, NULL, compress, 0, NULL); if ( !fs->nffile ) { args->done = 1; args->exit = 255; pthread_kill(args->parent, SIGUSR1); pthread_exit((void *)args); } fs->xstat = NULL; // init vars fs->bad_packets = 0; fs->first_seen = 0xffffffffffffLL; fs->last_seen = 0; t_start = 0; t_clock = 0; t_udp_flush = 0; while ( 1 ) { struct FlowNode *Node; Node = Pop_Node(args->NodeList, &args->done); if ( Node ) { t_clock = Node->t_last.tv_sec; dbg_printf("p_flow_thread() Next Node\n"); } else { done = args->done; dbg_printf("p_flow_thread() NULL Node\n"); } if ( t_start == 0 ) { t_udp_flush = t_start = t_clock - (t_clock % t_win); } if (((t_clock - t_start) >= t_win) || done) { /* rotate file */ struct tm *when; nffile_t *nffile; char FullName[MAXPATHLEN]; char netflowFname[128]; char error[256]; char *subdir; // flush all flows to disk DumpNodeStat(); uint32_t NumFlows = Flush_FlowTree(fs); when = localtime(&t_start); nffile = fs->nffile; // prepare sub dir hierarchy if ( subdir_index ) { subdir = GetSubDir(when); if ( !subdir ) { // failed to generate subdir path - put flows into base directory LogError("Failed to create subdir path!"); // failed to generate subdir path - put flows into base directory subdir = NULL; snprintf(netflowFname, 127, "nfcapd.%i%02i%02i%02i%02i", when->tm_year + 1900, when->tm_mon + 1, when->tm_mday, when->tm_hour, when->tm_min); } else { snprintf(netflowFname, 127, "%s/nfcapd.%i%02i%02i%02i%02i", subdir, when->tm_year + 1900, when->tm_mon + 1, when->tm_mday, when->tm_hour, when->tm_min); } } else { subdir = NULL; snprintf(netflowFname, 127, "nfcapd.%i%02i%02i%02i%02i", when->tm_year + 1900, when->tm_mon + 1, when->tm_mday, when->tm_hour, when->tm_min); } netflowFname[127] = '\0'; if ( subdir && !SetupSubDir(fs->datadir, subdir, error, 255) ) { // in this case the flows get lost! - the rename will fail // but this should not happen anyway, unless i/o problems, inode problems etc. LogError("Ident: %s, Failed to create sub hier directories: %s", fs->Ident, error ); } if ( nffile->block_header->NumRecords ) { // flush current buffer to disc if ( WriteBlock(nffile) <= 0 ) LogError("Ident: %s, failed to write output buffer to disk: '%s'" , fs->Ident, strerror(errno)); } // else - no new records in current block // prepare full filename snprintf(FullName, MAXPATHLEN-1, "%s/%s", fs->datadir, netflowFname); FullName[MAXPATHLEN-1] = '\0'; // update stat record // if no flows were collected, fs->last_seen is still 0 // set first_seen to start of this time slot, with twin window size. if ( fs->last_seen == 0 ) { fs->first_seen = (uint64_t)1000 * (uint64_t)t_start; fs->last_seen = (uint64_t)1000 * (uint64_t)(t_start + t_win); } nffile->stat_record->first_seen = fs->first_seen/1000; nffile->stat_record->msec_first = fs->first_seen - nffile->stat_record->first_seen*1000; nffile->stat_record->last_seen = fs->last_seen/1000; nffile->stat_record->msec_last = fs->last_seen - nffile->stat_record->last_seen*1000; // Flush Exporter Stat to file FlushExporterStats(fs); // Close file CloseUpdateFile(nffile, fs->Ident); // if rename fails, we are in big trouble, as we need to get rid of the old .current file // otherwise, we will loose flows and can not continue collecting new flows if ( !RenameAppend(fs->current, FullName) ) { LogError("Ident: %s, Can't rename dump file: %s", fs->Ident, strerror(errno)); LogError("Ident: %s, Serious Problem! Fix manually", fs->Ident); /* XXX if ( launcher_pid ) commbuff->failed = 1; */ // we do not update the books here, as the file failed to rename properly // otherwise the books may be wrong } else { struct stat fstat; /* XXX if ( launcher_pid ) commbuff->failed = 0; */ // Update books stat(FullName, &fstat); UpdateBooks(fs->bookkeeper, t_start, 512*fstat.st_blocks); } LogInfo("Ident: '%s' Flows: %llu, Packets: %llu, Bytes: %llu, Max Flows: %u", fs->Ident, (unsigned long long)nffile->stat_record->numflows, (unsigned long long)nffile->stat_record->numpackets, (unsigned long long)nffile->stat_record->numbytes, NumFlows); // reset stats fs->bad_packets = 0; fs->first_seen = 0xffffffffffffLL; fs->last_seen = 0; // Dump all extension maps and exporters to the buffer FlushStdRecords(fs); if ( done ) break; t_start = t_clock - (t_clock % t_win); nffile = OpenNewFile(fs->current, nffile, compress, 0, NULL); if ( !nffile ) { LogError("Fatal: OpenNewFile() failed for ident: %s", fs->Ident); args->done = 1; args->exit = 255; pthread_kill(args->parent, SIGUSR1); break; } } if (((t_clock - t_udp_flush) >= 10) || !done) { /* flush inactive UDP list */ UDPexpire(fs, t_clock - 10 ); t_udp_flush = t_clock; } if ( Node->fin != SIGNAL_NODE ) // Process the Node ProcessFlowNode(fs, Node); } while ( fs ) { DisposeFile(fs->nffile); fs = fs->next; } LogInfo("Terminating flow processng: exit: %i", args->exit); dbg_printf("End flow thread[%lu]\n", (long unsigned)args->tid); pthread_exit((void *)args); /* NOTREACHED */ } // End of p_flow_thread __attribute__((noreturn)) static void *p_pcap_flush_thread(void *thread_data) { // argument dispatching p_pcap_flush_thread_args_t *args = (p_pcap_flush_thread_args_t *)thread_data; char *pcap_datadir = args->pcap_datadir; pcap_dev_t *pcap_dev = args->pcap_dev; pcapfile_t *pcapfile = args->pcapfile; char pcap_dumpfile[MAXPATHLEN]; int err; int runs = 0; dbg_printf("New flush thread[%lu]\n", (long unsigned)args->tid); args->done = 0; args->exit = 0; err = pthread_setspecific( buffer_key, (void *)args ); if ( err ) { LogError("[%lu] pthread_setspecific() error in %s line %d: %s\n", (long unsigned)args->tid, __FILE__, __LINE__, strerror(errno) ); args->done = 1; args->exit = 255; pthread_kill(args->parent, SIGUSR1); pthread_exit((void *)args); } snprintf(pcap_dumpfile, MAXPATHLEN-1, "%s/%s.%lu", pcap_datadir , PCAP_DUMPFILE, (unsigned long)getpid() ); pcapfile = OpenNewPcapFile(pcap_dev->handle, pcap_dumpfile, pcapfile); if ( !pcapfile ) { args->done = 1; args->exit = 255; pthread_kill(args->parent, SIGUSR1); pthread_exit((void *)args); /* NOTREACHED */ } // wait for alternate buffer to be ready to flush while ( 1 ) { pthread_mutex_lock(&pcapfile->m_pbuff); while ( pcapfile->alternate_size == 0 && !args->done ) { pthread_cond_wait(&pcapfile->c_pbuff, &pcapfile->m_pbuff); } dbg_printf("Flush cycle\n"); runs++; // try to flush alternate buffer if ( pcapfile->alternate_size ) { // flush alternate buffer dbg_printf("Flush alternate\n"); if ( write(pcapfile->pfd, (void *)pcapfile->alternate_buffer, pcapfile->alternate_size) <= 0 ) { LogError("write() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); } pcapfile->alternate_size = 0; } // if we are done, try to flsuh main data buffer if ( args->done && pcapfile->data_size ) { dbg_printf("Done: Flush all buffers\n"); // flush alternate buffer if ( write(pcapfile->pfd, (void *)pcapfile->data_buffer, pcapfile->data_size) <= 0 ) { LogError("write() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); } pcapfile->data_size = 0; pcapfile->data_ptr = pcapfile->data_buffer; } // check if we need to rotate/close the file if ( args->done || pcapfile->t_CloseRename ) { /* rotate file */ struct tm *when; char FullName[MAXPATHLEN]; char pcapFname[128]; char error[256]; char *subdir; int err; dbg_printf("Flush rotate file\n"); when = localtime(&pcapfile->t_CloseRename); pcapfile->t_CloseRename = 0; // prepare sub dir hierarchy if ( args->subdir_index ) { subdir = GetSubDir(when); if ( !subdir ) { // failed to generate subdir path - put flows into base directory LogError("Failed to create subdir path!"); // failed to generate subdir path - put flows into base directory subdir = NULL; snprintf(pcapFname, 127, "pcapd.%i%02i%02i%02i%02i", when->tm_year + 1900, when->tm_mon + 1, when->tm_mday, when->tm_hour, when->tm_min); } else { snprintf(pcapFname, 127, "%s/pcapd.%i%02i%02i%02i%02i", subdir, when->tm_year + 1900, when->tm_mon + 1, when->tm_mday, when->tm_hour, when->tm_min); } } else { subdir = NULL; snprintf(pcapFname, 127, "pcapd.%i%02i%02i%02i%02i", when->tm_year + 1900, when->tm_mon + 1, when->tm_mday, when->tm_hour, when->tm_min); } pcapFname[127] = '\0'; if ( subdir && !SetupSubDir(pcap_datadir, subdir, error, 255) ) { // in this case the flows get lost! - the rename will fail // but this should not happen anyway, unless i/o problems, inode problems etc. LogError("p_packet_thread() Failed to create sub hier directories: %s", error ); } // prepare full filename snprintf(FullName, MAXPATHLEN-1, "%s/%s", pcap_datadir, pcapFname); FullName[MAXPATHLEN-1] = '\0'; ClosePcapFile(pcapfile); err = rename(pcap_dumpfile, FullName); if (err) { LogError("rename() pcap failed in %s line %d: %s", __FILE__, __LINE__, strerror(errno) ); } dbg_printf("Rotate file: %s -> %s\n", pcap_dumpfile, FullName); if ( args->done ) { pthread_mutex_unlock(&pcapfile->m_pbuff); pthread_cond_signal(&pcapfile->c_pbuff); break; } // open new files pcapfile = OpenNewPcapFile(pcap_dev->handle, pcap_dumpfile, pcapfile); if (!pcapfile) { args->done = 1; args->exit = 255; pthread_kill(args->parent, SIGUSR1); pthread_mutex_unlock(&pcapfile->m_pbuff); pthread_cond_signal(&pcapfile->c_pbuff); break; } } dbg_printf("Flush cycle done\n"); pthread_mutex_unlock(&pcapfile->m_pbuff); pthread_cond_signal(&pcapfile->c_pbuff); } dbg_printf("End flush thread[%lu]: %i runs\n", (long unsigned)args->tid, runs); pthread_exit((void *)args); /* NOTREACHED */ } // End of p_pcap_flush_thread __attribute__((noreturn)) static void *p_packet_thread(void *thread_data) { // argument dispatching p_packet_thread_args_t *args = (p_packet_thread_args_t *)thread_data; pcap_dev_t *pcap_dev = args->pcap_dev; time_t t_win = args->t_win; char *pcap_datadir = args->pcap_datadir; int subdir_index = args->subdir_index; int live = args->live; // locals p_pcap_flush_thread_args_t p_flush_thread_args; pcapfile_t *pcapfile; time_t t_start; int err; dbg_printf("New packet thread[%lu]\n", (long unsigned)args->tid); args->done = 0; args->exit = 0; err = pthread_setspecific( buffer_key, (void *)args ); if ( err ) { LogError("[%lu] pthread_setspecific() error in %s line %d: %s\n", (long unsigned)args->tid, __FILE__, __LINE__, strerror(errno) ); args->done = 1; args->exit = 255; pthread_kill(args->parent, SIGUSR1); pthread_exit((void *)args); /* NOTREACHED */ } /* start flush and pcap file handler thread */ if ( pcap_datadir ) { // just allocate pcapfile and buffers - we need to share pcapfile pcapfile = OpenNewPcapFile(pcap_dev->handle, NULL, NULL); if ( !pcapfile ) { args->done = 1; args->exit = 255; pthread_kill(args->parent, SIGUSR1); pthread_exit((void *)args); /* NOTREACHED */ } p_flush_thread_args.done = 0; p_flush_thread_args.exit = 0; p_flush_thread_args.parent = args->tid; p_flush_thread_args.pcap_dev = args->pcap_dev; p_flush_thread_args.subdir_index = subdir_index; p_flush_thread_args.pcap_datadir = pcap_datadir; p_flush_thread_args.pcapfile = pcapfile; err = pthread_create(&p_flush_thread_args.tid, NULL, p_pcap_flush_thread, (void *)&p_flush_thread_args); if ( err ) { LogError("pthread_create() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); args->done = 1; args->exit = 255; pthread_kill(args->parent, SIGUSR1); pthread_exit((void *)args); } dbg_printf("Started flush thread[%lu]\n", (long unsigned)p_flush_thread_args.tid); } else { pcapfile = NULL; } err = 0; t_start = 0; while ( 1 ) { struct pcap_pkthdr *hdr; const u_char *data; struct timeval tv; time_t t_clock; int ret; if ( !args->done ) { ret = pcap_next_ex(pcap_dev->handle, &hdr, &data); t_clock = 0; switch (ret) { case 1: { // packet read ok t_clock = hdr->ts.tv_sec; // process packet for flow cache ProcessPacket(args->NodeList, pcap_dev, hdr, data); if ( pcap_datadir ) { // keep the packet if (((t_clock - t_start) >= t_win)) { // first packet or rotate file if ( t_start != 0 ) { RotateFile(pcapfile, t_start, live); } // if first packet - set t_start here t_start = t_clock - (t_clock % t_win); } PcapDump(pcapfile, hdr, data); } } break; case 0: { // live capture idle cycle dbg_printf("pcap_next_ex() read live - timeout\n"); gettimeofday(&tv, NULL); t_clock = tv.tv_sec; if ((t_clock - t_start) >= t_win) { /* rotate file */ if ( t_start ) { // if not first packet, where t_start = 0 struct FlowNode *Node = New_Node(); Node->t_first = tv; Node->t_last = tv; Node->fin = SIGNAL_NODE; Push_Node(args->NodeList, Node); if ( pcap_datadir ) // keep the packet RotateFile(pcapfile, t_start, live); LogInfo("Packet processing stats: Total: %u, Skipped: %u, Unknown: %u, Short snaplen: %u", pcap_dev->proc_stat.packets, pcap_dev->proc_stat.skipped, pcap_dev->proc_stat.unknown, pcap_dev->proc_stat.short_snap); } t_start = t_clock - (t_clock % t_win); memset((void *)&(pcap_dev->proc_stat), 0, sizeof(proc_stat_t)); } } break; case -1: // signal error reading the packet err = 1; LogError("pcap_next_ex() read error: '%s'", pcap_geterr(pcap_dev->handle)); args->done = 1; continue; break; case -2: // End of packet file // signal parent, job is done err = 1; LogInfo("pcap_next_ex() end of file"); args->done = 1; LogInfo("Packet processing stats: Total: %u, Skipped: %u, Unknown: %u, Short snaplen: %u", pcap_dev->proc_stat.packets, pcap_dev->proc_stat.skipped, pcap_dev->proc_stat.unknown, pcap_dev->proc_stat.short_snap); continue; break; default: err = 1; pcap_breakloop(pcap_dev->handle); LogError("Unexpected pcap_next_ex() return value: %i", ret); args->done = 1; continue; } } if ( args->done ) break; } if ( pcap_datadir ) { dbg_printf("Wait for flush thread to complete\n"); pthread_mutex_lock(&pcapfile->m_pbuff); while ( pcapfile->alternate_size ) { pthread_cond_wait(&pcapfile->c_pbuff, &pcapfile->m_pbuff); } pcapfile->t_CloseRename = t_start; pthread_mutex_unlock(&pcapfile->m_pbuff); dbg_printf("Wait done.\n"); LogInfo("Signal flush thread[%lu] to terminate", p_flush_thread_args.tid); SignalThreadTerminate((thread_info_t *)&p_flush_thread_args, &pcapfile->c_pbuff); } if ( err ) pthread_kill(args->parent, SIGUSR1); LogInfo("Packet processing stats: Total: %u, Skipped: %u, Unknown: %u, Short snaplen: %u", pcap_dev->proc_stat.packets, pcap_dev->proc_stat.skipped, pcap_dev->proc_stat.unknown, pcap_dev->proc_stat.short_snap); LogInfo("Terminating packet dumping: exit: %i", args->exit); dbg_printf("End packet thread[%lu]\n", (long unsigned)args->tid); pthread_exit((void *)args); /* NOTREACHED */ } /* End of p_packet_thread */ static void WaitDone(void) { sigset_t signal_set; int done, sig; pthread_t tid = pthread_self(); LogInfo("[%lu] WaitDone() waiting", (long unsigned)tid); sigemptyset(&signal_set); sigaddset(&signal_set, SIGINT); sigaddset(&signal_set, SIGHUP); sigaddset(&signal_set, SIGTERM); sigaddset(&signal_set, SIGUSR1); pthread_sigmask(SIG_BLOCK, &signal_set, NULL); done = 0; while ( !done ) { sigwait(&signal_set, &sig); LogInfo("[%lu] WaitDone() signal %i", (long unsigned)tid, sig); switch ( sig ) { case SIGHUP: break; case SIGINT: case SIGTERM: pthread_mutex_lock(&m_done); done = 1; pthread_mutex_unlock(&m_done); pthread_cond_signal(&terminate); break; case SIGUSR1: // child signals end of job done = 1; break; // default: // empty } } } // End of WaitDone int main(int argc, char *argv[]) { sigset_t signal_set; struct sigaction sa; int c, snaplen, err, do_daemonize; int subdir_index, compress, expire, cache_size; FlowSource_t *fs; dirstat_t *dirstat; time_t t_win; char *device, *pcapfile, *filter, *datadir, *pcap_datadir, *extension_tags, pidfile[MAXPATHLEN], pidstr[32]; char *Ident, *userid, *groupid; pcap_dev_t *pcap_dev; p_packet_thread_args_t *p_packet_thread_args; p_flow_thread_args_t *p_flow_thread_args; snaplen = 100; do_daemonize = 0; launcher_pid = 0; device = NULL; pcapfile = NULL; filter = NULL; pidfile[0] = '\0'; t_win = TIME_WINDOW; datadir = DEFAULT_DIR; pcap_datadir = NULL; userid = groupid = NULL; Ident = "none"; fs = NULL; extension_tags = DefaultExtensions; subdir_index = 0; compress = 0; verbose = 0; expire = 0; cache_size = 0; while ((c = getopt(argc, argv, "B:DEI:g:hi:r:s:l:p:P:t:u:S:T:e:Vz")) != EOF) { switch (c) { struct stat fstat; case 'h': usage(argv[0]); exit(0); break; case 'u': userid = optarg; break; case 'g': groupid = optarg; break; case 'D': do_daemonize = 1; break; case 'B': cache_size = atoi(optarg); if (cache_size <= 0) { LogError("ERROR: Cache size must not be < 0"); exit(EXIT_FAILURE); } break; case 'I': Ident = strdup(optarg); break; case 'i': device = optarg; break; case 'l': datadir = optarg; err = stat(datadir, &fstat); if (!(fstat.st_mode & S_IFDIR)) { LogError("No such directory: " "'%s'", datadir); break; } break; case 'p': pcap_datadir = optarg; err = stat(pcap_datadir, &fstat); if (!(fstat.st_mode & S_IFDIR)) { LogError("No such directory: " "'%s'", pcap_datadir); break; } break; case 'r': { struct stat stat_buf; pcapfile = optarg; if ( stat(pcapfile, &stat_buf) ) { LogError("Can't stat '%s': %s", pcapfile, strerror(errno)); exit(EXIT_FAILURE); } if (!S_ISREG(stat_buf.st_mode) ) { LogError("'%s' is not a file", pcapfile); exit(EXIT_FAILURE); } } break; case 's': snaplen = atoi(optarg); if (snaplen < 14 + 20 + 20) { // ethernet, IP , TCP, no payload LogError("ERROR:, snaplen < sizeof IPv4 - Need 54 bytes for TCP/IPv4"); exit(EXIT_FAILURE); } break; case 't': t_win = atoi(optarg); if (t_win < 60) { LogError("WARNING, very small time frame (< 60)!"); } if (t_win <= 0) { LogError("ERROR: time frame too small (<= 0)"); exit(EXIT_FAILURE); } break; case 'z': compress = 1; break; case 'P': if ( optarg[0] == '/' ) { // absolute path given strncpy(pidfile, optarg, MAXPATHLEN-1); } else { // path relative to current working directory char tmp[MAXPATHLEN]; if ( !getcwd(tmp, MAXPATHLEN-1) ) { fprintf(stderr, "Failed to get current working directory: %s\n", strerror(errno)); exit(255); } tmp[MAXPATHLEN-1] = 0; snprintf(pidfile, MAXPATHLEN - 1 - strlen(tmp), "%s/%s", tmp, optarg); } // pidfile now absolute path pidfile[MAXPATHLEN-1] = 0; break; case 'S': subdir_index = atoi(optarg); break; case 'T': { size_t len = strlen(optarg); extension_tags = optarg; if ( len == 0 || len > 128 ) { fprintf(stderr, "Extension length error. Unexpected option '%s'\n", extension_tags); exit(255); } break; } case 'E': verbose = 1; Setv6Mode(1); break; case 'V': printf("%s: Version: %s\n",argv[0], nfdump_version); exit(0); break; default: usage(argv[0]); exit(EXIT_FAILURE); } } if (argc - optind > 1) { usage(argv[0]); exit(EXIT_FAILURE); } else { /* user specified a pcap filter */ filter = argv[optind]; } if ( fs == NULL && datadir != NULL && !AddDefaultFlowSource(&fs, Ident, datadir) ) { fprintf(stderr, "Failed to add default data collector directory\n"); exit(255); } if ( device && pcapfile ) { LogError("Specify either a device or a pcapfile, but not both"); exit(EXIT_FAILURE); } if ( !device && !pcapfile ) { LogError("Specify either a device or a pcapfile to read packets from"); exit(EXIT_FAILURE); } if ( !Init_FlowTree(cache_size)) { LogError("Init_FlowTree() failed."); exit(EXIT_FAILURE); } InitExtensionMaps(NO_EXTENSION_LIST); SetupExtensionDescriptors(strdup(extension_tags)); if ( pcapfile ) { pcap_dev = setup_pcap_file(pcapfile, filter, snaplen); } else { pcap_dev = setup_pcap_live(device, filter, snaplen); } if (!pcap_dev) { exit(EXIT_FAILURE); } SetPriv(userid, groupid); if ( subdir_index && !InitHierPath(subdir_index) ) { pcap_close(pcap_dev->handle); exit(255); } if ( do_daemonize && !InitLog(argv[0], SYSLOG_FACILITY)) { pcap_close(pcap_dev->handle); exit(255); } // check if pid file exists and if so, if a process with registered pid is running if ( strlen(pidfile) ) { int pidf; pidf = open(pidfile, O_RDONLY, 0); if ( pidf > 0 ) { // pid file exists char s[32]; ssize_t len; len = read(pidf, (void *)s, 31); close(pidf); s[31] = '\0'; if ( len < 0 ) { fprintf(stderr, "read() error existing pid file: %s\n", strerror(errno)); pcap_close(pcap_dev->handle); exit(255); } else { unsigned long pid = atol(s); if ( pid == 0 ) { // garbage - use this file unlink(pidfile); } else { if ( kill(pid, 0) == 0 ) { // process exists fprintf(stderr, "A process with pid %lu registered in pidfile %s is already running!\n", pid, strerror(errno)); pcap_close(pcap_dev->handle); exit(255); } else { // no such process - use this file unlink(pidfile); } } } } else { if ( errno != ENOENT ) { fprintf(stderr, "open() error existing pid file: %s\n", strerror(errno)); pcap_close(pcap_dev->handle); exit(255); } // else errno == ENOENT - no file - this is fine } } if ( do_daemonize ) { verbose = 0; daemonize(); } if (strlen(pidfile)) { pid_t pid = getpid(); int pidf = open(pidfile, O_RDWR|O_TRUNC|O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if ( pidf == -1 ) { LogError("Error opening pid file: '%s' %s", pidfile, strerror(errno)); pcap_close(pcap_dev->handle); exit(255); } snprintf(pidstr,31,"%lu\n", (unsigned long)pid); if ( write(pidf, pidstr, strlen(pidstr)) <= 0 ) { LogError("Error write pid file: '%s' %s", pidfile, strerror(errno)); } close(pidf); } if ( InitBookkeeper(&fs->bookkeeper, fs->datadir, getpid(), launcher_pid) != BOOKKEEPER_OK ) { LogError("initialize bookkeeper failed."); pcap_close(pcap_dev->handle); exit(255); } // Init the extension map list if ( !InitExtensionMapList(fs) ) { // error message goes to syslog pcap_close(pcap_dev->handle); exit(255); } IPFragTree_init(); LogInfo("Startup."); // prepare signal mask for all threads // block signals, as they are handled by the main thread // mask is inherited by all threads sigemptyset(&signal_set); sigaddset(&signal_set, SIGINT); sigaddset(&signal_set, SIGHUP); sigaddset(&signal_set, SIGTERM); sigaddset(&signal_set, SIGUSR1); sigaddset(&signal_set, SIGPIPE); pthread_sigmask(SIG_BLOCK, &signal_set, NULL); // for USR2 set a signal handler, which interrupts blocking // system calls - and signals done event // handler applies for all threads in a process sa.sa_handler = Interrupt_handler; sigemptyset(&sa.sa_mask); sa.sa_flags = 0; sigaction(SIGPIPE, &sa, NULL); sigaction(SIGUSR2, &sa, NULL); // key for each thread err = pthread_key_create(&buffer_key, NULL); if ( err ) { LogError("pthread_key() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); exit(255); } // prepare flow thread args p_flow_thread_args = (p_flow_thread_args_t *)malloc(sizeof(p_flow_thread_args_t)); if ( !p_flow_thread_args ) { LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); exit(255); } p_flow_thread_args->fs = fs; p_flow_thread_args->t_win = t_win; p_flow_thread_args->compress = compress; p_flow_thread_args->subdir_index = subdir_index; p_flow_thread_args->parent = pthread_self(); p_flow_thread_args->NodeList = NewNodeList(); err = 0; err = pthread_create(&p_flow_thread_args->tid, NULL, p_flow_thread, (void *)p_flow_thread_args); if ( err ) { LogError("pthread_create() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); exit(255); } dbg_printf("Started flow thread[%lu]\n", (long unsigned)p_flow_thread_args->tid); // prepare packet thread args p_packet_thread_args = (p_packet_thread_args_t *)malloc(sizeof(p_packet_thread_args_t)); if ( !p_packet_thread_args ) { LogError("malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); exit(255); } p_packet_thread_args->pcap_dev = pcap_dev; p_packet_thread_args->t_win = t_win; p_packet_thread_args->subdir_index = subdir_index; p_packet_thread_args->pcap_datadir = pcap_datadir; p_packet_thread_args->live = device != NULL; p_packet_thread_args->parent = pthread_self(); p_packet_thread_args->NodeList = p_flow_thread_args->NodeList; err = pthread_create(&p_packet_thread_args->tid, NULL, p_packet_thread, (void *)p_packet_thread_args); if ( err ) { LogError("pthread_create() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) ); exit(255); } dbg_printf("Started packet thread[%lu]\n", (long unsigned)p_packet_thread_args->tid); // Wait till done WaitDone(); dbg_printf("Signal packet thread to terminate\n"); SignalThreadTerminate((thread_info_t *)p_packet_thread_args, NULL); dbg_printf("Signal flow thread to terminate\n"); SignalThreadTerminate((thread_info_t *)p_flow_thread_args, &p_packet_thread_args->NodeList->c_list); // free arg list free((void *)p_packet_thread_args); free((void *)p_flow_thread_args); LogInfo("Terminating nfpcapd."); if ( expire == 0 && ReadStatInfo(fs->datadir, &dirstat, LOCK_IF_EXISTS) == STATFILE_OK ) { UpdateBookStat(dirstat, fs->bookkeeper); WriteStatInfo(dirstat); LogInfo("Updating statinfo in directory '%s'", datadir); } ReleaseBookkeeper(fs->bookkeeper, DESTROY_BOOKKEEPER); pcap_close(pcap_dev->handle); if ( strlen(pidfile) ) unlink(pidfile); EndLog(); exit(EXIT_SUCCESS); } /* End of main */