suricata
source-erf-file.c
Go to the documentation of this file.
1 /* Copyright (C) 2010-2014 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Endace Technology Limited.
22  *
23  * Support for reading ERF files.
24  *
25  * Only ethernet supported at this time.
26  */
27 
28 #include "suricata-common.h"
29 #include "suricata.h"
30 #include "tm-threads.h"
31 #include "source-erf-file.h"
32 #include "util-datalink.h"
33 
34 #define DAG_TYPE_ETH 2
35 
36 typedef struct DagFlags_ {
37  uint8_t iface:2;
38  uint8_t vlen:1;
39  uint8_t trunc:1;
40  uint8_t rxerror:1;
41  uint8_t dserror:1;
42  uint8_t reserved:1;
43  uint8_t direction:1;
45 
46 typedef struct DagRecord_ {
47  uint64_t ts;
48  uint8_t type;
50  uint16_t rlen;
51  uint16_t lctr;
52  uint16_t wlen;
53  uint16_t pad;
54 } __attribute__((packed)) DagRecord;
55 
56 typedef struct ErfFileThreadVars_ {
58  TmSlot *slot;
59 
60  FILE *erf;
61 
62  uint32_t pkts;
63  uint64_t bytes;
65 
66 static inline TmEcode ReadErfRecord(ThreadVars *, Packet *, void *);
67 TmEcode ReceiveErfFileLoop(ThreadVars *, void *, void *);
68 TmEcode ReceiveErfFileThreadInit(ThreadVars *, const void *, void **);
71 
72 static TmEcode DecodeErfFileThreadInit(ThreadVars *, const void *, void **);
73 static TmEcode DecodeErfFileThreadDeinit(ThreadVars *tv, void *data);
74 static TmEcode DecodeErfFile(ThreadVars *, Packet *, void *);
75 
76 /**
77  * \brief Register the ERF file receiver (reader) module.
78  */
79 void
81 {
82  tmm_modules[TMM_RECEIVEERFFILE].name = "ReceiveErfFile";
92 }
93 
94 /**
95  * \brief Register the ERF file decoder module.
96  */
97 void
99 {
100  tmm_modules[TMM_DECODEERFFILE].name = "DecodeErfFile";
101  tmm_modules[TMM_DECODEERFFILE].ThreadInit = DecodeErfFileThreadInit;
102  tmm_modules[TMM_DECODEERFFILE].Func = DecodeErfFile;
104  tmm_modules[TMM_DECODEERFFILE].ThreadDeinit = DecodeErfFileThreadDeinit;
107 }
108 
109 /**
110  * \brief ERF file reading loop.
111  */
112 TmEcode ReceiveErfFileLoop(ThreadVars *tv, void *data, void *slot)
113 {
114  Packet *p = NULL;
115  ErfFileThreadVars *etv = (ErfFileThreadVars *)data;
116 
117  etv->slot = ((TmSlot *)slot)->slot_next;
118 
119  // Indicate that the thread is actually running its application level code (i.e., it can poll
120  // packets)
122 
123  while (1) {
126  }
127 
128  /* Make sure we have at least one packet in the packet pool,
129  * to prevent us from alloc'ing packets at line rate. */
130  PacketPoolWait();
131 
133  if (unlikely(p == NULL)) {
134  SCLogError("Failed to allocate a packet.");
135  EngineStop();
137  }
139 
140  if (ReadErfRecord(tv, p, data) != TM_ECODE_OK) {
141  TmqhOutputPacketpool(etv->tv, p);
142  EngineStop();
144  }
145 
146  if (TmThreadsSlotProcessPkt(etv->tv, etv->slot, p) != TM_ECODE_OK) {
147  EngineStop();
149  }
150  }
152 }
153 
154 static inline TmEcode ReadErfRecord(ThreadVars *tv, Packet *p, void *data)
155 {
156  SCEnter();
157 
158  ErfFileThreadVars *etv = (ErfFileThreadVars *)data;
159  DagRecord dr;
160 
161  int r = fread(&dr, sizeof(DagRecord), 1, etv->erf);
162  if (r < 1) {
163  if (feof(etv->erf)) {
164  SCLogInfo("End of ERF file reached");
165  }
166  else {
167  SCLogInfo("Error reading ERF record");
168  }
170  }
171  uint16_t rlen = SCNtohs(dr.rlen);
172  uint16_t wlen = SCNtohs(dr.wlen);
173  if (rlen < sizeof(DagRecord)) {
174  SCLogError("Bad ERF record, "
175  "record length less than size of header");
177  }
178  r = fread(GET_PKT_DATA(p), rlen - sizeof(DagRecord), 1, etv->erf);
179  if (r < 1) {
180  if (feof(etv->erf)) {
181  SCLogInfo("End of ERF file reached");
182  }
183  else {
184  SCLogInfo("Error reading ERF record");
185  }
187  }
188 
189  /* Only support ethernet at this time. */
190  if (dr.type != DAG_TYPE_ETH) {
191  SCLogError("DAG record type %d not implemented.", dr.type);
193  }
194 
195  GET_PKT_LEN(p) = wlen;
197 
198  /* Convert ERF time to timeval - from libpcap. */
199  uint64_t ts = dr.ts;
200  p->ts = SCTIME_FROM_SECS(ts >> 32);
201  ts = (ts & 0xffffffffULL) * 1000000;
202  ts += 0x80000000; /* rounding */
203  uint64_t usecs = (ts >> 32);
204  if (usecs >= 1000000) {
205  usecs -= 1000000;
206  p->ts = SCTIME_ADD_SECS(p->ts, 1);
207  usecs++;
208  }
209  p->ts = SCTIME_ADD_USECS(p->ts, usecs);
210 
211  etv->pkts++;
212  etv->bytes += wlen;
213 
215 }
216 
217 /**
218  * \brief Initialize the ERF receiver thread.
219  */
220 TmEcode
221 ReceiveErfFileThreadInit(ThreadVars *tv, const void *initdata, void **data)
222 {
223  SCEnter();
224 
225  if (initdata == NULL) {
226  SCLogError("Error: No filename provided.");
228  }
229 
230  FILE *erf = fopen((const char *)initdata, "r");
231  if (erf == NULL) {
232  SCLogError("Failed to open %s: %s", (char *)initdata, strerror(errno));
233  exit(EXIT_FAILURE);
234  }
235 
237  if (unlikely(etv == NULL)) {
238  SCLogError("Failed to allocate memory for ERF file thread vars.");
239  fclose(erf);
241  }
242  memset(etv, 0, sizeof(*etv));
243  etv->erf = erf;
244  etv->tv = tv;
245  *data = (void *)etv;
246 
247  SCLogInfo("Processing ERF file %s", (char *)initdata);
248 
250 
252 }
253 
254 /**
255  * \brief Initialize the ERF decoder thread.
256  */
257 TmEcode
258 DecodeErfFileThreadInit(ThreadVars *tv, const void *initdata, void **data)
259 {
260  SCEnter();
261  DecodeThreadVars *dtv = NULL;
263 
264  if (dtv == NULL)
266 
268 
269  *data = (void *)dtv;
270 
272 }
273 
274 TmEcode DecodeErfFileThreadDeinit(ThreadVars *tv, void *data)
275 {
276  if (data != NULL)
277  DecodeThreadVarsFree(tv, data);
279 }
280 
281 /**
282  * \brief Decode the ERF file.
283  *
284  * This function ups the decoder counters and then passes the packet
285  * off to the ethernet decoder.
286  */
287 TmEcode
288 DecodeErfFile(ThreadVars *tv, Packet *p, void *data)
289 {
290  SCEnter();
292 
294 
295  /* Update counters. */
297 
299 
301 
303 }
304 
305 /**
306  * \brief Print some stats to the log at program exit.
307  *
308  * \param tv Pointer to ThreadVars.
309  * \param data Pointer to data, ErfFileThreadVars.
310  */
311 void
313 {
314  ErfFileThreadVars *etv = (ErfFileThreadVars *)data;
315 
316  SCLogInfo("Packets: %"PRIu32"; Bytes: %"PRIu64, etv->pkts, etv->bytes);
317 }
TmModule_::cap_flags
uint8_t cap_flags
Definition: tm-modules.h:67
TMM_RECEIVEERFFILE
@ TMM_RECEIVEERFFILE
Definition: tm-threads-common.h:50
tm-threads.h
DagRecord_::type
uint8_t type
Definition: source-erf-file.c:48
ts
uint64_t ts
Definition: source-erf-file.c:55
DagRecord_::lctr
uint16_t lctr
Definition: source-erf-file.c:51
DAG_TYPE_ETH
#define DAG_TYPE_ETH
Definition: source-erf-file.c:34
PKT_IS_PSEUDOPKT
#define PKT_IS_PSEUDOPKT(p)
return 1 if the packet is a pseudo packet
Definition: decode.h:1059
DagRecord_::ts
uint64_t ts
Definition: source-erf-file.c:47
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
TmThreadsSetFlag
void TmThreadsSetFlag(ThreadVars *tv, uint32_t flag)
Set a thread flag.
Definition: tm-threads.c:99
DagFlags_::trunc
uint8_t trunc
Definition: source-erf-file.c:39
ReceiveErfFileThreadExitStats
void ReceiveErfFileThreadExitStats(ThreadVars *, void *)
Print some stats to the log at program exit.
Definition: source-erf-file.c:312
DagRecord_::flags
DagFlags flags
Definition: source-erf-file.c:49
TmModuleDecodeErfFileRegister
void TmModuleDecodeErfFileRegister(void)
Register the ERF file decoder module.
Definition: source-erf-file.c:98
THV_RUNNING
#define THV_RUNNING
Definition: threadvars.h:54
SURICATA_STOP
#define SURICATA_STOP
Definition: suricata.h:89
PacketDecodeFinalize
void PacketDecodeFinalize(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p)
Finalize decoding of a packet.
Definition: decode.c:147
TmqhOutputPacketpool
void TmqhOutputPacketpool(ThreadVars *t, Packet *p)
Definition: tmqh-packetpool.c:356
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:85
TmModule_::PktAcqLoop
TmEcode(* PktAcqLoop)(ThreadVars *, void *, void *)
Definition: tm-modules.h:54
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:84
TmModule_::ThreadDeinit
TmEcode(* ThreadDeinit)(ThreadVars *, void *)
Definition: tm-modules.h:49
Packet_::datalink
int datalink
Definition: decode.h:607
PKT_SET_SRC
#define PKT_SET_SRC(p, src_val)
Definition: decode.h:1062
DecodeRegisterPerfCounters
void DecodeRegisterPerfCounters(DecodeThreadVars *dtv, ThreadVars *tv)
Definition: decode.c:526
SCTIME_FROM_SECS
#define SCTIME_FROM_SECS(s)
Definition: util-time.h:55
PKT_SRC_WIRE
@ PKT_SRC_WIRE
Definition: decode.h:54
DagFlags_::rxerror
uint8_t rxerror
Definition: source-erf-file.c:40
SCTIME_ADD_USECS
#define SCTIME_ADD_USECS(ts, us)
Definition: util-time.h:54
DagFlags_::direction
uint8_t direction
Definition: source-erf-file.c:43
TmModule_::PktAcqBreakLoop
TmEcode(* PktAcqBreakLoop)(ThreadVars *, void *)
Definition: tm-modules.h:57
Packet_::ts
SCTime_t ts
Definition: decode.h:471
SCEnter
#define SCEnter(...)
Definition: util-debug.h:271
GET_PKT_DATA
#define GET_PKT_DATA(p)
Definition: decode.h:219
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:57
TmModule_::Func
TmEcode(* Func)(ThreadVars *, Packet *, void *)
Definition: tm-modules.h:52
DagRecord_::rlen
uint16_t rlen
Definition: source-erf-file.c:50
DagFlags_
Definition: source-erf-file.c:36
ErfFileThreadVars
ErfFileThreadVars
Definition: source-erf-file.c:64
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:289
PacketPoolWait
void PacketPoolWait(void)
Definition: tmqh-packetpool.c:69
Packet_
Definition: decode.h:428
TM_FLAG_DECODE_TM
#define TM_FLAG_DECODE_TM
Definition: tm-modules.h:32
tmm_modules
TmModule tmm_modules[TMM_SIZE]
Definition: tm-modules.c:33
wlen
uint16_t wlen
Definition: source-erf-file.c:60
GET_PKT_LEN
#define GET_PKT_LEN(p)
Definition: decode.h:218
TmSlot_
Definition: tm-threads.h:53
DagFlags
struct DagFlags_ DagFlags
TmEcode
TmEcode
Definition: tm-threads-common.h:83
TMM_DECODEERFFILE
@ TMM_DECODEERFFILE
Definition: tm-threads-common.h:51
TmModule_::name
const char * name
Definition: tm-modules.h:44
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:224
TM_FLAG_RECEIVE_TM
#define TM_FLAG_RECEIVE_TM
Definition: tm-modules.h:31
TmModuleReceiveErfFileRegister
void TmModuleReceiveErfFileRegister(void)
Register the ERF file receiver (reader) module.
Definition: source-erf-file.c:80
dtv
DecodeThreadVars * dtv
Definition: fuzz_decodepcapfile.c:33
ReceiveErfFileThreadDeinit
TmEcode ReceiveErfFileThreadDeinit(ThreadVars *, void *)
__attribute__
struct DagRecord_ __attribute__((packed))
DNP3 application object header.
Definition: source-erf-file.c:54
DecodeThreadVarsFree
void DecodeThreadVarsFree(ThreadVars *tv, DecodeThreadVars *dtv)
Definition: decode.c:708
SCNtohs
#define SCNtohs(x)
Definition: suricata-common.h:403
suricata-common.h
DagFlags_::dserror
uint8_t dserror
Definition: source-erf-file.c:41
TmModule_::ThreadInit
TmEcode(* ThreadInit)(ThreadVars *, const void *, void **)
Definition: tm-modules.h:47
EngineStop
void EngineStop(void)
make sure threads can stop the engine by calling this function. Purpose: pcap file mode needs to be a...
Definition: suricata.c:427
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
TmModule_::ThreadExitPrintStats
void(* ThreadExitPrintStats)(ThreadVars *, void *)
Definition: tm-modules.h:48
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:261
DagFlags_::vlen
uint8_t vlen
Definition: source-erf-file.c:38
DagRecord_::wlen
uint16_t wlen
Definition: source-erf-file.c:52
DecodeThreadVars_
Structure to hold thread specific data for all decode modules.
Definition: decode.h:664
DagFlags_::reserved
uint8_t reserved
Definition: source-erf-file.c:42
source-erf-file.h
DecodeThreadVarsAlloc
DecodeThreadVars * DecodeThreadVarsAlloc(ThreadVars *tv)
Alloc and setup DecodeThreadVars.
Definition: decode.c:689
DagFlags_::iface
uint8_t iface
Definition: source-erf-file.c:37
DagRecord_
Definition: source-erf-file.c:46
suricata.h
DagRecord_::pad
uint16_t pad
Definition: source-erf-file.c:53
ReceiveErfFileThreadInit
TmEcode ReceiveErfFileThreadInit(ThreadVars *, const void *, void **)
Initialize the ERF receiver thread.
Definition: source-erf-file.c:221
SCTIME_ADD_SECS
#define SCTIME_ADD_SECS(ts, s)
Definition: util-time.h:53
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
ReceiveErfFileLoop
TmEcode ReceiveErfFileLoop(ThreadVars *, void *, void *)
ERF file reading loop.
Definition: source-erf-file.c:112
PacketGetFromQueueOrAlloc
Packet * PacketGetFromQueueOrAlloc(void)
Get a packet. We try to get a packet from the packetpool first, but if that is empty we alloc a packe...
Definition: decode.c:208
DecodeEthernet
int DecodeEthernet(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *pkt, uint32_t len)
Definition: decode-ethernet.c:42
TmModule_::flags
uint8_t flags
Definition: tm-modules.h:70
DecodeUpdatePacketCounters
void DecodeUpdatePacketCounters(ThreadVars *tv, const DecodeThreadVars *dtv, const Packet *p)
Definition: decode.c:655
LINKTYPE_ETHERNET
#define LINKTYPE_ETHERNET
Definition: decode.h:972
suricata_ctl_flags
volatile uint8_t suricata_ctl_flags
Definition: suricata.c:164
rlen
uint16_t rlen
Definition: source-erf-file.c:58