suricata
app-layer-expectation.c
Go to the documentation of this file.
1 /* Copyright (C) 2017-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \defgroup applayerexpectation Application Layer Expectation
20  *
21  * Handling of dynamic parallel connection for application layer similar
22  * to FTP.
23  *
24  * @{
25  *
26  * Some protocols like FTP create dynamic parallel flow (called expectation). In
27  * order to assign a application layer protocol to these expectation, Suricata
28  * needs to parse message of the initial protocol and create and maintain a list
29  * of expected flow.
30  *
31  * Application layers must use the here described API to implement this mechanism.
32  *
33  * When parsing a application layer message describing a parallel flow, the
34  * application layer can call AppLayerExpectationCreate() to declare an
35  * expectation. By doing that the next flow coming with corresponding IP parameters
36  * will be assigned the specified application layer. The resulting Flow will
37  * also have a Flow storage set that can be retrieved at index
38  * AppLayerExpectationGetDataId():
39  *
40  * ```
41  * data = (char *)FlowGetStorageById(f, AppLayerExpectationGetDataId());
42  * ```
43  * This storage can be used to store information that are only available in the
44  * parent connection and could be useful in the parent connection. For instance
45  * this is used by the FTP protocol to propagate information such as file name
46  * and ftp operation to the FTP data connection.
47  */
48 
49 /**
50  * \file
51  *
52  * \author Eric Leblond <eric@regit.org>
53  */
54 
55 #include "suricata-common.h"
56 #include "debug.h"
57 
58 #include "ippair-storage.h"
59 #include "flow-storage.h"
60 
61 #include "app-layer-expectation.h"
62 
63 #include "util-print.h"
64 #include "queue.h"
65 
66 static int g_expectation_id = -1;
67 static int g_expectation_data_id = -1;
68 
69 SC_ATOMIC_DECLARE(uint32_t, expectation_count);
70 
71 #define EXPECTATION_TIMEOUT 30
72 #define EXPECTATION_MAX_LEVEL 10
73 
74 typedef struct Expectation_ {
75  struct timeval ts;
79  int direction;
80  /* use pointer to Flow as identifier of the Flow the expectation is linked to */
81  void *orig_f;
82  void *data;
83  CIRCLEQ_ENTRY(Expectation_) entries;
85 
86 typedef struct ExpectationData_ {
87  /** Start of Expectation Data structure must be a pointer
88  * to free function. Set to NULL to use SCFree() */
89  void (*DFree)(void *);
91 
92 typedef struct ExpectationList_ {
93  CIRCLEQ_HEAD(EList, Expectation_) list;
94  uint8_t length;
96 
97 static void ExpectationDataFree(void *e)
98 {
99  SCLogDebug("Free expectation data");
100  ExpectationData *ed = (ExpectationData *) e;
101  if (ed->DFree) {
102  ed->DFree(e);
103  } else {
104  SCFree(e);
105  }
106 }
107 
108 /**
109  * Free expectation
110  */
111 static void AppLayerFreeExpectation(Expectation *exp)
112 {
113  if (exp->data) {
114  ExpectationData *expdata = (ExpectationData *)exp->data;
115  if (expdata->DFree) {
116  expdata->DFree(exp->data);
117  } else {
118  SCFree(exp->data);
119  }
120  }
121  SCFree(exp);
122 }
123 
124 static void ExpectationListFree(void *el)
125 {
126  ExpectationList *exp_list = (ExpectationList *)el;
127  Expectation *exp, *pexp;
128  if (exp_list == NULL)
129  return;
130 
131  if (exp_list->length > 0) {
132  CIRCLEQ_FOREACH_SAFE(exp, &exp_list->list, entries, pexp) {
133  CIRCLEQ_REMOVE(&exp_list->list, exp, entries);
134  exp_list->length--;
135  AppLayerFreeExpectation(exp);
136  }
137  }
138  SCFree(exp_list);
139 }
140 
141 uint64_t ExpectationGetCounter(void)
142 {
143  uint64_t x = SC_ATOMIC_GET(expectation_count);
144  return x;
145 }
146 
148 {
149  g_expectation_id = IPPairStorageRegister("expectation", sizeof(void *), NULL, ExpectationListFree);
150  g_expectation_data_id = FlowStorageRegister("expectation", sizeof(void *), NULL, ExpectationDataFree);
151  SC_ATOMIC_INIT(expectation_count);
152 }
153 
154 static inline int GetFlowAddresses(Flow *f, Address *ip_src, Address *ip_dst)
155 {
156  memset(ip_src, 0, sizeof(*ip_src));
157  memset(ip_dst, 0, sizeof(*ip_dst));
158  if (FLOW_IS_IPV4(f)) {
159  FLOW_COPY_IPV4_ADDR_TO_PACKET(&f->src, ip_src);
160  FLOW_COPY_IPV4_ADDR_TO_PACKET(&f->dst, ip_dst);
161  } else if (FLOW_IS_IPV6(f)) {
162  FLOW_COPY_IPV6_ADDR_TO_PACKET(&f->src, ip_src);
163  FLOW_COPY_IPV6_ADDR_TO_PACKET(&f->dst, ip_dst);
164  } else {
165  return -1;
166  }
167  return 0;
168 }
169 
170 static ExpectationList *AppLayerExpectationLookup(Flow *f, IPPair **ipp)
171 {
172  Address ip_src, ip_dst;
173  if (GetFlowAddresses(f, &ip_src, &ip_dst) == -1)
174  return NULL;
175  *ipp = IPPairLookupIPPairFromHash(&ip_src, &ip_dst);
176  if (*ipp == NULL) {
177  return NULL;
178  }
179 
180  return IPPairGetStorageById(*ipp, g_expectation_id);
181 }
182 
183 
184 static ExpectationList *AppLayerExpectationRemove(IPPair *ipp,
185  ExpectationList *exp_list,
186  Expectation *exp)
187 {
188  CIRCLEQ_REMOVE(&exp_list->list, exp, entries);
189  AppLayerFreeExpectation(exp);
190  SC_ATOMIC_SUB(expectation_count, 1);
191  exp_list->length--;
192  if (exp_list->length == 0) {
193  IPPairSetStorageById(ipp, g_expectation_id, NULL);
194  ExpectationListFree(exp_list);
195  exp_list = NULL;
196  }
197  return exp_list;
198 }
199 
200 /**
201  * Create an entry in expectation list
202  *
203  * Create a expectation from an existing Flow. Currently, only Flow between
204  * the two original IP addresses are supported. In case of success, the
205  * ownership of the data pointer is taken. In case of error, the pointer
206  * to data has to be freed by the caller.
207  *
208  * \param f a pointer to the original Flow
209  * \param direction the direction of the data in the expectation flow
210  * \param src source port of the expected flow, use 0 for any
211  * \param dst destination port of the expected flow, use 0 for any
212  * \param alproto the protocol that need to be set on the expected flow
213  * \param data pointer to data that will be attached to the expected flow
214  *
215  * \return -1 if error
216  * \return 0 if success
217  */
218 int AppLayerExpectationCreate(Flow *f, int direction, Port src, Port dst,
219  AppProto alproto, void *data)
220 {
221  ExpectationList *exp_list = NULL;
222  IPPair *ipp;
223  Address ip_src, ip_dst;
224 
225  Expectation *exp = SCCalloc(1, sizeof(*exp));
226  if (exp == NULL)
227  return -1;
228 
229  exp->sp = src;
230  exp->dp = dst;
231  exp->alproto = alproto;
232  exp->ts = f->lastts;
233  exp->orig_f = (void *)f;
234  exp->data = data;
235  exp->direction = direction;
236 
237  if (GetFlowAddresses(f, &ip_src, &ip_dst) == -1)
238  goto error;
239  ipp = IPPairGetIPPairFromHash(&ip_src, &ip_dst);
240  if (ipp == NULL)
241  goto error;
242 
243  exp_list = IPPairGetStorageById(ipp, g_expectation_id);
244  if (exp_list) {
245  CIRCLEQ_INSERT_HEAD(&exp_list->list, exp, entries);
246  /* In case there is already EXPECTATION_MAX_LEVEL expectations waiting to be fullfill,
247  * we remove the older expectation to limit the total number of expectations */
248  if (exp_list->length >= EXPECTATION_MAX_LEVEL) {
249  Expectation *last_exp = CIRCLEQ_LAST(&exp_list->list);
250  CIRCLEQ_REMOVE(&exp_list->list, last_exp, entries);
251  AppLayerFreeExpectation(last_exp);
252  /* We keep the same amount of expectation so we fully release
253  * the IP pair */
255  IPPairRelease(ipp);
256  return 0;
257  }
258  } else {
259  exp_list = SCCalloc(1, sizeof(*exp_list));
260  if (exp_list == NULL)
261  goto error;
262  exp_list->length = 0;
263  CIRCLEQ_INIT(&exp_list->list);
264  CIRCLEQ_INSERT_HEAD(&exp_list->list, exp, entries);
265  IPPairSetStorageById(ipp, g_expectation_id, exp_list);
266  }
267 
268  exp_list->length += 1;
269  SC_ATOMIC_ADD(expectation_count, 1);
271  /* As we are creating the expectation, we release lock on IPPair without
272  * setting the ref count to 0. This way the IPPair will be kept till
273  * cleanup */
274  IPPairUnlock(ipp);
275  return 0;
276 
277 error:
278  SCFree(exp);
279  return -1;
280 }
281 
282 /**
283  * Return Flow storage identifier corresponding to expectation data
284  *
285  * \return expectation data identifier
286  */
288 {
289  return g_expectation_data_id;
290 }
291 
292 /**
293  * Function doing a lookup in expectation list and updating Flow if needed.
294  *
295  * This function lookup for a existing expectation that could match the Flow.
296  * If found and if the expectation contains data it store the data in the
297  * expectation storage of the Flow.
298  *
299  * \return an AppProto value if found
300  * \return ALPROTO_UNKNOWN if not found
301  */
303 {
304  AppProto alproto = ALPROTO_UNKNOWN;
305  IPPair *ipp = NULL;
306  Expectation *lexp = NULL;
307  Expectation *exp = NULL;
308 
309  int x = SC_ATOMIC_GET(expectation_count);
310  if (x == 0) {
311  return ALPROTO_UNKNOWN;
312  }
313 
314  /* Call will take reference of the ip pair in 'ipp' */
315  ExpectationList *exp_list = AppLayerExpectationLookup(f, &ipp);
316  if (exp_list == NULL)
317  goto out;
318 
319  time_t ctime = f->lastts.tv_sec;
320 
321  CIRCLEQ_FOREACH_SAFE(exp, &exp_list->list, entries, lexp) {
322  if ((exp->direction & direction) &&
323  ((exp->sp == 0) || (exp->sp == f->sp)) &&
324  ((exp->dp == 0) || (exp->dp == f->dp))) {
325  alproto = exp->alproto;
326  f->alproto_ts = alproto;
327  f->alproto_tc = alproto;
328  void *fdata = FlowGetStorageById(f, g_expectation_id);
329  if (fdata) {
330  /* We already have an expectation so let's clean this one */
331  ExpectationDataFree(exp->data);
332  } else {
333  /* Transfer ownership of Expectation data to the Flow */
334  if (FlowSetStorageById(f, g_expectation_data_id, exp->data) != 0) {
335  SCLogDebug("Unable to set flow storage");
336  }
337  }
338  exp->data = NULL;
339  exp_list = AppLayerExpectationRemove(ipp, exp_list, exp);
340  if (exp_list == NULL)
341  goto out;
342  continue;
343  }
344  /* Cleaning remove old entries */
345  if (ctime > exp->ts.tv_sec + EXPECTATION_TIMEOUT) {
346  exp_list = AppLayerExpectationRemove(ipp, exp_list, exp);
347  if (exp_list == NULL)
348  goto out;
349  continue;
350  }
351  }
352 
353 out:
354  if (ipp)
355  IPPairRelease(ipp);
356  return alproto;
357 }
358 
360 {
361  IPPair *ipp = NULL;
362  Expectation *exp = NULL;
363  Expectation *pexp = NULL;
364 
365  int x = SC_ATOMIC_GET(expectation_count);
366  if (x == 0) {
367  return;
368  }
369 
370  /* Call will take reference of the ip pair in 'ipp' */
371  ExpectationList *exp_list = AppLayerExpectationLookup(f, &ipp);
372  if (exp_list == NULL)
373  goto out;
374 
375  CIRCLEQ_FOREACH_SAFE(exp, &exp_list->list, entries, pexp) {
376  /* Cleaning remove old entries */
377  if (exp->orig_f == (void *)f) {
378  exp_list = AppLayerExpectationRemove(ipp, exp_list, exp);
379  if (exp_list == NULL)
380  goto out;
381  }
382  }
383 
384 out:
385  if (ipp)
386  IPPairRelease(ipp);
387  return;
388 }
389 
390 /**
391  * @}
392  */
EXPECTATION_TIMEOUT
#define EXPECTATION_TIMEOUT
Definition: app-layer-expectation.c:71
Expectation_::alproto
AppProto alproto
Definition: app-layer-expectation.c:78
FLOW_HAS_EXPECTATION
#define FLOW_HAS_EXPECTATION
Definition: flow.h:109
CIRCLEQ_FOREACH_SAFE
#define CIRCLEQ_FOREACH_SAFE(var, head, field, tvar)
Definition: queue.h:553
ExpectationGetCounter
uint64_t ExpectationGetCounter(void)
Definition: app-layer-expectation.c:141
FLOW_IS_IPV6
#define FLOW_IS_IPV6(f)
Definition: flow.h:154
Expectation_::data
void * data
Definition: app-layer-expectation.c:82
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:303
AppLayerExpectationSetup
void AppLayerExpectationSetup(void)
Definition: app-layer-expectation.c:147
IPPairRelease
void IPPairRelease(IPPair *h)
Definition: ippair.c:518
CIRCLEQ_HEAD
#define CIRCLEQ_HEAD(name, type)
Definition: queue.h:439
AppLayerExpectationHandle
AppProto AppLayerExpectationHandle(Flow *f, int direction)
Definition: app-layer-expectation.c:302
Expectation_::orig_f
void * orig_f
Definition: app-layer-expectation.c:81
Expectation
struct Expectation_ Expectation
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
ExpectationData_::DFree
void(* DFree)(void *)
Definition: app-layer-expectation.c:89
AppProto
uint16_t AppProto
Definition: app-layer-protos.h:73
Flow_
Flow data structure.
Definition: flow.h:347
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:321
Expectation_::ts
struct timeval ts
Definition: app-layer-expectation.c:75
Address_
Definition: decode.h:114
ExpectationData
struct ExpectationData_ ExpectationData
ExpectationData_
Definition: app-layer-expectation.c:86
Flow_::dp
Port dp
Definition: flow.h:359
AppLayerExpectationGetDataId
int AppLayerExpectationGetDataId(void)
Definition: app-layer-expectation.c:287
AppLayerExpectationClean
void AppLayerExpectationClean(Flow *f)
Definition: app-layer-expectation.c:359
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint32_t, expectation_count)
FLOW_COPY_IPV6_ADDR_TO_PACKET
#define FLOW_COPY_IPV6_ADDR_TO_PACKET(fa, pa)
Definition: flow.h:167
Flow_::dst
FlowAddress dst
Definition: flow.h:350
Expectation_::sp
Port sp
Definition: app-layer-expectation.c:76
app-layer-expectation.h
FlowStorageRegister
int FlowStorageRegister(const char *name, const unsigned int size, void *(*Alloc)(unsigned int), void(*Free)(void *))
Definition: flow-storage.c:65
ExpectationList
struct ExpectationList_ ExpectationList
util-print.h
IPPairGetIPPairFromHash
IPPair * IPPairGetIPPairFromHash(Address *a, Address *b)
Definition: ippair.c:542
FLOW_IS_IPV4
#define FLOW_IS_IPV4(f)
Definition: flow.h:152
IPPairUnlock
void IPPairUnlock(IPPair *h)
Definition: ippair.c:529
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:330
EXPECTATION_MAX_LEVEL
#define EXPECTATION_MAX_LEVEL
Definition: app-layer-expectation.c:72
AppLayerExpectationCreate
int AppLayerExpectationCreate(Flow *f, int direction, Port src, Port dst, AppProto alproto, void *data)
Definition: app-layer-expectation.c:218
CIRCLEQ_ENTRY
#define CIRCLEQ_ENTRY(type)
Definition: queue.h:448
Port
uint16_t Port
Definition: decode.h:239
queue.h
FlowSetStorageById
int FlowSetStorageById(Flow *f, int id, void *ptr)
Definition: flow-storage.c:44
IPPairStorageRegister
int IPPairStorageRegister(const char *name, const unsigned int size, void *(*Alloc)(unsigned int), void(*Free)(void *))
Definition: ippair-storage.c:61
ExpectationList_
Definition: app-layer-expectation.c:92
Flow_::src
FlowAddress src
Definition: flow.h:350
Flow_::lastts
struct timeval lastts
Definition: flow.h:404
IPPairGetStorageById
void * IPPairGetStorageById(IPPair *h, int id)
Definition: ippair-storage.c:35
length
uint16_t length
Definition: app-layer-modbus.c:2
flow-storage.h
suricata-common.h
IPPair_
Definition: ippair.h:58
CIRCLEQ_REMOVE
#define CIRCLEQ_REMOVE(head, elm, field)
Definition: queue.h:523
CIRCLEQ_INSERT_HEAD
#define CIRCLEQ_INSERT_HEAD(head, elm, field)
Definition: queue.h:503
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:451
CIRCLEQ_INIT
#define CIRCLEQ_INIT(head)
Definition: queue.h:478
Flow_::flags
uint32_t flags
Definition: flow.h:421
src
uint16_t src
Definition: app-layer-dnp3.h:5
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
FlowGetStorageById
void * FlowGetStorageById(Flow *f, int id)
Definition: flow-storage.c:39
IPPairLookupIPPairFromHash
IPPair * IPPairLookupIPPairFromHash(Address *a, Address *b)
look up a ippair in the hash
Definition: ippair.c:641
Expectation_::dp
Port dp
Definition: app-layer-expectation.c:77
Flow_::sp
Port sp
Definition: flow.h:352
dst
uint16_t dst
Definition: app-layer-dnp3.h:4
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:364
IPPairSetStorageById
int IPPairSetStorageById(IPPair *h, int id, void *ptr)
Definition: ippair-storage.c:40
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:452
ippair-storage.h
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
Expectation_
Definition: app-layer-expectation.c:74
debug.h
Expectation_::direction
int direction
Definition: app-layer-expectation.c:79
FLOW_COPY_IPV4_ADDR_TO_PACKET
#define FLOW_COPY_IPV4_ADDR_TO_PACKET(fa, pa)
Definition: flow.h:162
CIRCLEQ_LAST
#define CIRCLEQ_LAST(head)
Definition: queue.h:458