farrokhi/nfdump
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Activate flowlabel in filter syntax

Browse Source
This commit is contained in:
Peter Haag 2017-12-29 15:02:00 +01:00
parent 746d1bcd47
commit fccabf69ab
14 changed files with 224 additions and 2639 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ChangeLog
View File

@ -1,6 +1,8 @@
2017-12-28
- Add sampling elements ID 302,304,305. put them identcal to ID 48,49,50
- Add option to label filter terms. syntax: ( <filter> ) %labelname.
- Add option to label filter terms. syntax: (<filter>) %labelname.
- Add %lbl option to print flow label in output
- Update nfdump(1) man page for flowlabels
2017-12-27
- Add ipfix delta timestamp elements 158/159.

16
bin/grammar.y
View File

@ -2099,7 +2099,21 @@ expr: term { $$ = $1.self; }
| '(' expr ')' { $$ = $2; }
| '(' expr ')' '%' STRING {
$$ = $2;
AddLabel($2, $5);
if ( strlen($5) > 16 ) {
yyerror("Error: Maximum 16 chars allowed for flowlabel");
YYABORT;
} else {
AddLabel($2, $5);
}
}
| '%' STRING '(' expr ')' {
$$ = $4;
if ( strlen($2) > 16 ) {
yyerror("Error: Maximum 16 chars allowed for flowlabel");
YYABORT;
} else {
AddLabel($4, $2);
}
}
;

19
bin/nf_common.c
View File

@ -211,6 +211,8 @@ static void String_MPLSs(master_record_t *r, char *string);
static void String_Engine(master_record_t *r, char *string);
static void String_Label(master_record_t *r, char *string);
static void String_ClientLatency(master_record_t *r, char *string);
static void String_ServerLatency(master_record_t *r, char *string);
@ -341,6 +343,7 @@ static struct format_token_list_s {
{ "%pps", 0, " pps", String_pps }, // pps - packets per second
{ "%bpp", 0, " Bpp", String_bpp }, // bpp - Bytes per package
{ "%eng", 0, " engine", String_Engine }, // Engine Type/ID
{ "%lbl", 0, " label", String_Label }, // Flow Label
#ifdef NSEL
// NSEL specifics
@ -768,6 +771,7 @@ extension_map_t *extension_map = r->map_ref;
snprintf(_s, slen-1, "\n"
"Flow Record: \n"
" Flags = 0x%.2x %s, %s\n"
" label = %16s\n"
" export sysid = %5u\n"
" size = %5u\n"
" first = %10u [%s]\n"
@ -778,7 +782,9 @@ extension_map_t *extension_map = r->map_ref;
" dst addr = %16s\n"
,
r->flags, TestFlag(r->flags, FLAG_EVENT) ? "EVENT" : "FLOW",
TestFlag(r->flags, FLAG_SAMPLED) ? "Sampled" : "Unsampled", r->exporter_sysid, r->size, r->first,
TestFlag(r->flags, FLAG_SAMPLED) ? "Sampled" : "Unsampled",
r->label ? r->label : "<none>",
r->exporter_sysid, r->size, r->first,
datestr1, r->last, datestr2, r->msec_first, r->msec_last,
as, ds );
@ -2547,6 +2553,17 @@ static void String_Engine(master_record_t *r, char *string) {
} // End of String_Engine
static void String_Label(master_record_t *r, char *string) {
if ( r->label )
snprintf(string, MAX_STRING_LENGTH-1 ,"%16s", r->label);
else
snprintf(string, MAX_STRING_LENGTH-1 ,"<none>");
string[MAX_STRING_LENGTH-1] = '\0';
} // End of String_Label
static void String_ClientLatency(master_record_t *r, char *string) {
double latency;

5
bin/nfdump.c
View File

@ -593,6 +593,11 @@ int v1_map_done = 0;
// Records passed filter -> continue record processing
// Update statistics
master_record->label = Engine->label;
#ifdef DEVEL
if ( Engine->label )
printf("Flow has label: %s\n", Engine->label);
#endif
UpdateStat(&stat_record, master_record);
// update number of flows matching a given map

102
bin/nfdump.test.diff
View File

@ -2,187 +2,187 @@
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
5c5
6c6
< size = 196
---
> size = 172
57c57
58c58
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
59c59
61c61
< size = 196
---
> size = 172
111c111
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
113c113
< size = 196
---
> size = 172
165c165
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
167c167
116c116
< size = 196
---
> size = 172
219c219
168c168
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
221c221
171c171
< size = 196
---
> size = 172
273c273
223c223
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
275c275
226c226
< size = 196
---
> size = 172
327c327
278c278
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
329c329
281c281
< size = 196
---
> size = 172
381c381
333c333
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
383c383
336c336
< size = 196
---
> size = 172
435c435
388c388
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
437c437
391c391
< size = 196
---
> size = 172
489c489
443c443
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
491c491
446c446
< size = 196
---
> size = 172
543c543
498c498
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
545c545
501c501
< size = 196
---
> size = 172
597c597
553c553
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
599c599
556c556
< size = 196
---
> size = 172
651c651
608c608
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
653c653
611c611
< size = 196
---
> size = 172
705c705
663c663
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
707c707
666c666
< size = 196
---
> size = 172
758c758
718c718
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
760c760
721c721
< size = 196
---
> size = 172
812c812
772c772
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
775c775
< size = 196
---
> size = 172
827c827
< Flags = 0x07 FLOW, Unsampled
---
> Flags = 0x01 FLOW, Unsampled
814c814
830c830
< size = 220
---
> size = 196
866c866
882c882
< Flags = 0x07 FLOW, Unsampled
---
> Flags = 0x01 FLOW, Unsampled
868c868
885c885
< size = 220
---
> size = 196
920c920
937c937
< Flags = 0x07 FLOW, Unsampled
---
> Flags = 0x05 FLOW, Unsampled
922c922
940c940
< size = 220
---
> size = 200
974c974
992c992
< Flags = 0x07 FLOW, Unsampled
---
> Flags = 0x03 FLOW, Unsampled
976c976
995c995
< size = 220
---
> size = 200
1030c1030
1050c1050
< size = 220
---
> size = 204
1082c1082
1102c1102
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x04 FLOW, Unsampled
1084c1084
1105c1105
< size = 196
---
> size = 176
1136c1136
1157c1157
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x02 FLOW, Unsampled
1138c1138
1160c1160
< size = 196
---
> size = 176
1192c1192
1215c1215
< size = 196
---
> size = 180
1246c1246
1270c1270
< size = 200
---
> size = 184
1300c1300
1325c1325
< size = 200
---
> size = 184
1354c1354
1380c1380
< size = 204
---
> size = 188

26
bin/nfdump.test.out
View File

@ -1,6 +1,7 @@
Flow Record:
Flags = 0x00 FLOW, Unsampled
label = <none>
export sysid = 1
size = 172
first = 1089534600 [2004-07-11 10:30:00]
@ -55,6 +56,7 @@ Flow Record:
Flow Record:
Flags = 0x00 FLOW, Unsampled
label = <none>
export sysid = 1
size = 172
first = 1089534610 [2004-07-11 10:30:10]
@ -109,6 +111,7 @@ Flow Record:
Flow Record:
Flags = 0x00 FLOW, Unsampled
label = <none>
export sysid = 1
size = 172
first = 1089534620 [2004-07-11 10:30:20]
@ -163,6 +166,7 @@ Flow Record:
Flow Record:
Flags = 0x00 FLOW, Unsampled
label = <none>
export sysid = 1
size = 172
first = 1089534630 [2004-07-11 10:30:30]
@ -217,6 +221,7 @@ Flow Record:
Flow Record:
Flags = 0x00 FLOW, Unsampled
label = <none>
export sysid = 1
size = 172
first = 1089534640 [2004-07-11 10:30:40]
@ -271,6 +276,7 @@ Flow Record:
Flow Record:
Flags = 0x00 FLOW, Unsampled
label = <none>
export sysid = 1
size = 172
first = 1089534650 [2004-07-11 10:30:50]
@ -325,6 +331,7 @@ Flow Record:
Flow Record:
Flags = 0x00 FLOW, Unsampled
label = <none>
export sysid = 1
size = 172
first = 1089534660 [2004-07-11 10:31:00]
@ -379,6 +386,7 @@ Flow Record:
Flow Record:
Flags = 0x00 FLOW, Unsampled
label = <none>
export sysid = 1
size = 172
first = 1089534670 [2004-07-11 10:31:10]
@ -433,6 +441,7 @@ Flow Record:
Flow Record:
Flags = 0x00 FLOW, Unsampled
label = <none>
export sysid = 1
size = 172
first = 1089534680 [2004-07-11 10:31:20]
@ -487,6 +496,7 @@ Flow Record:
Flow Record:
Flags = 0x00 FLOW, Unsampled
label = <none>
export sysid = 1
size = 172
first = 1089534690 [2004-07-11 10:31:30]
@ -541,6 +551,7 @@ Flow Record:
Flow Record:
Flags = 0x00 FLOW, Unsampled
label = <none>
export sysid = 1
size = 172
first = 1089534700 [2004-07-11 10:31:40]
@ -595,6 +606,7 @@ Flow Record:
Flow Record:
Flags = 0x00 FLOW, Unsampled
label = <none>
export sysid = 1
size = 172
first = 1089534710 [2004-07-11 10:31:50]
@ -649,6 +661,7 @@ Flow Record:
Flow Record:
Flags = 0x00 FLOW, Unsampled
label = <none>
export sysid = 1
size = 172
first = 1089534720 [2004-07-11 10:32:00]
@ -703,6 +716,7 @@ Flow Record:
Flow Record:
Flags = 0x00 FLOW, Unsampled
label = <none>
export sysid = 1
size = 172
first = 1089534730 [2004-07-11 10:32:10]
@ -756,6 +770,7 @@ Flow Record:
Flow Record:
Flags = 0x00 FLOW, Unsampled
label = <none>
export sysid = 1
size = 172
first = 1089534740 [2004-07-11 10:32:20]
@ -810,6 +825,7 @@ Flow Record:
Flow Record:
Flags = 0x01 FLOW, Unsampled
label = <none>
export sysid = 1
size = 196
first = 1089534750 [2004-07-11 10:32:30]
@ -864,6 +880,7 @@ Flow Record:
Flow Record:
Flags = 0x01 FLOW, Unsampled
label = <none>
export sysid = 1
size = 196
first = 1089534760 [2004-07-11 10:32:40]
@ -918,6 +935,7 @@ Flow Record:
Flow Record:
Flags = 0x05 FLOW, Unsampled
label = <none>
export sysid = 1
size = 200
first = 1089534770 [2004-07-11 10:32:50]
@ -972,6 +990,7 @@ Flow Record:
Flow Record:
Flags = 0x03 FLOW, Unsampled
label = <none>
export sysid = 1
size = 200
first = 1089534780 [2004-07-11 10:33:00]
@ -1026,6 +1045,7 @@ Flow Record:
Flow Record:
Flags = 0x07 FLOW, Unsampled
label = <none>
export sysid = 1
size = 204
first = 1089534790 [2004-07-11 10:33:10]
@ -1080,6 +1100,7 @@ Flow Record:
Flow Record:
Flags = 0x04 FLOW, Unsampled
label = <none>
export sysid = 1
size = 176
first = 1089534800 [2004-07-11 10:33:20]
@ -1134,6 +1155,7 @@ Flow Record:
Flow Record:
Flags = 0x02 FLOW, Unsampled
label = <none>
export sysid = 1
size = 176
first = 1089534810 [2004-07-11 10:33:30]
@ -1188,6 +1210,7 @@ Flow Record:
Flow Record:
Flags = 0x06 FLOW, Unsampled
label = <none>
export sysid = 1
size = 180
first = 1089534820 [2004-07-11 10:33:40]
@ -1242,6 +1265,7 @@ Flow Record:
Flow Record:
Flags = 0x06 FLOW, Unsampled
label = <none>
export sysid = 1
size = 184
first = 1089534830 [2004-07-11 10:33:50]
@ -1296,6 +1320,7 @@ Flow Record:
Flow Record:
Flags = 0x06 FLOW, Unsampled
label = <none>
export sysid = 1
size = 184
first = 1089534840 [2004-07-11 10:34:00]
@ -1350,6 +1375,7 @@ Flow Record:
Flow Record:
Flags = 0x06 FLOW, Unsampled
label = <none>
export sysid = 1
size = 188
first = 1089534850 [2004-07-11 10:34:10]

3
bin/nffile.h
View File

@ -2102,6 +2102,9 @@ typedef struct master_record_s {
// last entry in master record
# define Offset_MR_LAST offsetof(master_record_t, map_ref)
extension_map_t *map_ref;
// optional flowlabel
char *label;
} master_record_t;
#define AnyMask 0xffffffffffffffffLL

1
bin/nftest.c
View File

@ -441,6 +441,7 @@ void *p;
flow_record.V4.dstaddr = 0x0a0a0a0b;
ret = check_filter_block("src ip 172.32.7.16", &flow_record, 1);
ret = check_filter_block("( src ip 172.32.7.16 ) %MyLabel", &flow_record, 1);
ret = check_filter_block("%MyLabel( src ip 172.32.7.16 )", &flow_record, 1);
ret = check_filter_block("src ip 172.32.7.15", &flow_record, 0);
ret = check_filter_block("dst ip 10.10.10.11", &flow_record, 1);
ret = check_filter_block("dst ip 10.10.10.10", &flow_record, 0);

28
bin/nftree.c
View File

@ -431,6 +431,7 @@ int RunFilter(FilterEngine_data_t *args) {
uint32_t index, offset;
int evaluate, invert;
args->label = NULL;
index = args->StartNode;
evaluate = 0;
invert = 0;
@ -450,6 +451,7 @@ uint32_t index, offset;
uint64_t comp_value[2];
int evaluate, invert;
args->label = NULL;
index = args->StartNode;
evaluate = 0;
invert = 0;
@ -497,14 +499,38 @@ int evaluate, invert;
break;
}
index = evaluate ? args->filter[index].OnTrue : args->filter[index].OnFalse;
/*
* Label evaluation:
* A flow gets labeled, if one filter expression has a label assigned and
* that filter expression is in the 'true' path of the tree, resulting
* to a final match. If subsequent expressions in the same path evaluate
* to false, the label is cleared again.
* In case of multiple labels in a true patch, the last seen label wins.
*/
if ( evaluate ) {
// if filter expression has a label assigned, copy that
if ( args->filter[index].label ) {
args->label = args->filter[index].label;
}
index = args->filter[index].OnTrue;
} else {
// filter expression does not match - clear previous label if abailable
if ( args->label )
args->label = NULL;
index = args->filter[index].OnFalse;
}
// index = evaluate ? args->filter[index].OnTrue : args->filter[index].OnFalse;
}
return invert ? !evaluate : evaluate;
} /* End of RunExtendedFilter */
void AddLabel(uint32_t index, char *label) {
FilterTree[index].label = strdup(label);
//Evaluation requires extended engine
Extended = 1;
} // End of AddLabel
uint32_t AddIdent(char *Ident) {

1
bin/nftree.h
View File

@ -75,6 +75,7 @@ typedef struct FilterEngine_data_s {
uint32_t Extended;
char **IdentList;
uint64_t *nfrecord;
char *label;
int (*FilterEngine)(struct FilterEngine_data_s *);
} FilterEngine_data_t;

2532
bin/out-test
View File

File diff suppressed because it is too large Load Diff

2
bin/test.sh
View File

@ -94,7 +94,7 @@ fi
# supress 'received at' as this is always different
./nfdump -r tmp/nfcapd.* -q -o raw | grep -v 'received at' > test5.out
# nfdump 1.6.5 always uses 64 bits. therefore we have a predictable diff
# nfdump 1.6.5 and later always use 64 bits. therefore we have a predictable diff
# so diff the diff
diff test5.out nfdump.test.out > test5.diff || true
diff test5.diff nfdump.test.diff

102
bin/test5.diff
View File

@ -2,187 +2,187 @@
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
5c5
6c6
< size = 196
---
> size = 172
57c57
58c58
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
59c59
61c61
< size = 196
---
> size = 172
111c111
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
113c113
< size = 196
---
> size = 172
165c165
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
167c167
116c116
< size = 196
---
> size = 172
219c219
168c168
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
221c221
171c171
< size = 196
---
> size = 172
273c273
223c223
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
275c275
226c226
< size = 196
---
> size = 172
327c327
278c278
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
329c329
281c281
< size = 196
---
> size = 172
381c381
333c333
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
383c383
336c336
< size = 196
---
> size = 172
435c435
388c388
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
437c437
391c391
< size = 196
---
> size = 172
489c489
443c443
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
491c491
446c446
< size = 196
---
> size = 172
543c543
498c498
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
545c545
501c501
< size = 196
---
> size = 172
597c597
553c553
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
599c599
556c556
< size = 196
---
> size = 172
651c651
608c608
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
653c653
611c611
< size = 196
---
> size = 172
705c705
663c663
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
707c707
666c666
< size = 196
---
> size = 172
758c758
718c718
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
760c760
721c721
< size = 196
---
> size = 172
812c812
772c772
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x00 FLOW, Unsampled
775c775
< size = 196
---
> size = 172
827c827
< Flags = 0x07 FLOW, Unsampled
---
> Flags = 0x01 FLOW, Unsampled
814c814
830c830
< size = 220
---
> size = 196
866c866
882c882
< Flags = 0x07 FLOW, Unsampled
---
> Flags = 0x01 FLOW, Unsampled
868c868
885c885
< size = 220
---
> size = 196
920c920
937c937
< Flags = 0x07 FLOW, Unsampled
---
> Flags = 0x05 FLOW, Unsampled
922c922
940c940
< size = 220
---
> size = 200
974c974
992c992
< Flags = 0x07 FLOW, Unsampled
---
> Flags = 0x03 FLOW, Unsampled
976c976
995c995
< size = 220
---
> size = 200
1030c1030
1050c1050
< size = 220
---
> size = 204
1082c1082
1102c1102
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x04 FLOW, Unsampled
1084c1084
1105c1105
< size = 196
---
> size = 176
1136c1136
1157c1157
< Flags = 0x06 FLOW, Unsampled
---
> Flags = 0x02 FLOW, Unsampled
1138c1138
1160c1160
< size = 196
---
> size = 176
1192c1192
1215c1215
< size = 196
---
> size = 180
1246c1246
1270c1270
< size = 200
---
> size = 184
1300c1300
1325c1325
< size = 200
---
> size = 184
1354c1354
1380c1380
< size = 204
---
> size = 188

22
man/nfdump.1
View File

@ -621,6 +621,8 @@ and format specifier as described below
.br
\fB%eng\fR Engine Type/ID
.br
\fB%lbl\fR Flowlabel
.br
\fB%sa\fR Source Address
.br
\fB%da\fR Destination Address
@ -1267,6 +1269,26 @@ Select the vrf
.RE
.PD
.SH "Flowlabel"
One or more specific filter expressions can be assigned a flowlabel in order to identify
the flow in the output according to the label. A flowlabel has the form \fB%LabelName\fR and is
appended or prepended to a filter expression in braces. It may have up to 16 characters.
Example: \fB(ip 8.8.8.8) %GoogleDNS\fR. If a filter matches, with a labeled expressions,
and that expression is in the matching filter patch, the label can be printed in the output,
using the \fB%%lbl\fR format token. See OUTPUT FORMATS.
Example: Add flowlabel to end of 'line' format:
.br
\fB./nfdump -r <file> -o 'fmt:%line %lbl" ..\fR
.br
Note: A filter may have multiple matching paths - for example \fBproto tcp or ip 8.8.8.8\fR
The shortest path which evaluates successfully, wins. Other paths are skipped, which means
that flowlabels are not printed in not evaluated filter paths. A filter may contain multiple
flowlabels. The flowlabel of the last matching expression in the winning path is printed.
Flowlabels are most useful in large and complex filters stored in one or multiple files,
to better read the flow output list.
.br
Example: \fB(ip in [172.16.1.0/24]) %ISP_1 or (ip in [172.16.16.0/24]) %IPS_2 or %GoogleDNS((proto udp or proto tcp) and ip 8.8.8.8)
.br
.SH "EXAMPLES"
.B nfdump \-r /and/dir/nfcapd.201107110845 \-c 100 'proto tcp and ( src ip 172.16.17.18 or dst ip 172.16.17.19 )'
Dumps the first 100 netflow records which match the given filter: