suricata
app-layer-expectation.c
Go to the documentation of this file.
1 /* Copyright (C) 2017-2021 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \defgroup applayerexpectation Application Layer Expectation
20  *
21  * Handling of dynamic parallel connection for application layer similar
22  * to FTP.
23  *
24  * @{
25  *
26  * Some protocols like FTP create dynamic parallel flow (called expectation). In
27  * order to assign a application layer protocol to these expectation, Suricata
28  * needs to parse message of the initial protocol and create and maintain a list
29  * of expected flow.
30  *
31  * Application layers must use the here described API to implement this mechanism.
32  *
33  * When parsing a application layer message describing a parallel flow, the
34  * application layer can call AppLayerExpectationCreate() to declare an
35  * expectation. By doing that the next flow coming with corresponding IP parameters
36  * will be assigned the specified application layer. The resulting Flow will
37  * also have a Flow storage set that can be retrieved at index
38  * AppLayerExpectationGetDataId():
39  *
40  * ```
41  * data = (char *)FlowGetStorageById(f, AppLayerExpectationGetFlowId());
42  * ```
43  * This storage can be used to store information that are only available in the
44  * parent connection and could be useful in the parent connection. For instance
45  * this is used by the FTP protocol to propagate information such as file name
46  * and ftp operation to the FTP data connection.
47  */
48 
49 /**
50  * \file
51  *
52  * \author Eric Leblond <eric@regit.org>
53  */
54 
55 #include "suricata-common.h"
56 #include "debug.h"
57 
58 #include "ippair-storage.h"
59 #include "flow-storage.h"
60 
61 #include "app-layer-expectation.h"
62 
63 #include "util-print.h"
64 #include "queue.h"
65 
66 static int g_ippair_expectation_id = -1;
67 static FlowStorageId g_flow_expectation_id = { .id = -1 };
68 
69 SC_ATOMIC_DECLARE(uint32_t, expectation_count);
70 
71 #define EXPECTATION_TIMEOUT 30
72 #define EXPECTATION_MAX_LEVEL 10
73 
74 typedef struct Expectation_ {
75  struct timeval ts;
79  int direction;
80  /* use pointer to Flow as identifier of the Flow the expectation is linked to */
81  void *orig_f;
82  void *data;
83  CIRCLEQ_ENTRY(Expectation_) entries;
85 
86 typedef struct ExpectationData_ {
87  /** Start of Expectation Data structure must be a pointer
88  * to free function. Set to NULL to use SCFree() */
89  void (*DFree)(void *);
91 
92 typedef struct ExpectationList_ {
93  CIRCLEQ_HEAD(EList, Expectation_) list;
94  uint8_t length;
96 
97 static void ExpectationDataFree(void *e)
98 {
99  SCLogDebug("Free expectation data");
100  ExpectationData *ed = (ExpectationData *) e;
101  if (ed->DFree) {
102  ed->DFree(e);
103  } else {
104  SCFree(e);
105  }
106 }
107 
108 /**
109  * Free expectation
110  */
111 static void AppLayerFreeExpectation(Expectation *exp)
112 {
113  if (exp->data) {
114  ExpectationData *expdata = (ExpectationData *)exp->data;
115  if (expdata->DFree) {
116  expdata->DFree(exp->data);
117  } else {
118  SCFree(exp->data);
119  }
120  }
121  SCFree(exp);
122 }
123 
124 static void ExpectationListFree(void *el)
125 {
126  ExpectationList *exp_list = (ExpectationList *)el;
127  Expectation *exp, *pexp;
128  if (exp_list == NULL)
129  return;
130 
131  if (exp_list->length > 0) {
132  CIRCLEQ_FOREACH_SAFE(exp, &exp_list->list, entries, pexp) {
133  CIRCLEQ_REMOVE(&exp_list->list, exp, entries);
134  exp_list->length--;
135  AppLayerFreeExpectation(exp);
136  }
137  }
138  SCFree(exp_list);
139 }
140 
141 uint64_t ExpectationGetCounter(void)
142 {
143  uint64_t x = SC_ATOMIC_GET(expectation_count);
144  return x;
145 }
146 
148 {
149  g_ippair_expectation_id =
150  IPPairStorageRegister("expectation", sizeof(void *), NULL, ExpectationListFree);
151  g_flow_expectation_id =
152  FlowStorageRegister("expectation", sizeof(void *), NULL, ExpectationDataFree);
153  SC_ATOMIC_INIT(expectation_count);
154 }
155 
156 static inline int GetFlowAddresses(Flow *f, Address *ip_src, Address *ip_dst)
157 {
158  memset(ip_src, 0, sizeof(*ip_src));
159  memset(ip_dst, 0, sizeof(*ip_dst));
160  if (FLOW_IS_IPV4(f)) {
161  FLOW_COPY_IPV4_ADDR_TO_PACKET(&f->src, ip_src);
162  FLOW_COPY_IPV4_ADDR_TO_PACKET(&f->dst, ip_dst);
163  } else if (FLOW_IS_IPV6(f)) {
164  FLOW_COPY_IPV6_ADDR_TO_PACKET(&f->src, ip_src);
165  FLOW_COPY_IPV6_ADDR_TO_PACKET(&f->dst, ip_dst);
166  } else {
167  return -1;
168  }
169  return 0;
170 }
171 
172 static ExpectationList *AppLayerExpectationLookup(Flow *f, IPPair **ipp)
173 {
174  Address ip_src, ip_dst;
175  if (GetFlowAddresses(f, &ip_src, &ip_dst) == -1)
176  return NULL;
177  *ipp = IPPairLookupIPPairFromHash(&ip_src, &ip_dst);
178  if (*ipp == NULL) {
179  return NULL;
180  }
181 
182  return IPPairGetStorageById(*ipp, g_ippair_expectation_id);
183 }
184 
185 
186 static ExpectationList *AppLayerExpectationRemove(IPPair *ipp,
187  ExpectationList *exp_list,
188  Expectation *exp)
189 {
190  CIRCLEQ_REMOVE(&exp_list->list, exp, entries);
191  AppLayerFreeExpectation(exp);
192  SC_ATOMIC_SUB(expectation_count, 1);
193  exp_list->length--;
194  if (exp_list->length == 0) {
195  IPPairSetStorageById(ipp, g_ippair_expectation_id, NULL);
196  ExpectationListFree(exp_list);
197  exp_list = NULL;
198  }
199  return exp_list;
200 }
201 
202 /**
203  * Create an entry in expectation list
204  *
205  * Create a expectation from an existing Flow. Currently, only Flow between
206  * the two original IP addresses are supported. In case of success, the
207  * ownership of the data pointer is taken. In case of error, the pointer
208  * to data has to be freed by the caller.
209  *
210  * \param f a pointer to the original Flow
211  * \param direction the direction of the data in the expectation flow
212  * \param src source port of the expected flow, use 0 for any
213  * \param dst destination port of the expected flow, use 0 for any
214  * \param alproto the protocol that need to be set on the expected flow
215  * \param data pointer to data that will be attached to the expected flow
216  *
217  * \return -1 if error
218  * \return 0 if success
219  */
220 int AppLayerExpectationCreate(Flow *f, int direction, Port src, Port dst,
221  AppProto alproto, void *data)
222 {
223  ExpectationList *exp_list = NULL;
224  IPPair *ipp;
225  Address ip_src, ip_dst;
226 
227  Expectation *exp = SCCalloc(1, sizeof(*exp));
228  if (exp == NULL)
229  return -1;
230 
231  exp->sp = src;
232  exp->dp = dst;
233  exp->alproto = alproto;
234  exp->ts = f->lastts;
235  exp->orig_f = (void *)f;
236  exp->data = data;
237  exp->direction = direction;
238 
239  if (GetFlowAddresses(f, &ip_src, &ip_dst) == -1)
240  goto error;
241  ipp = IPPairGetIPPairFromHash(&ip_src, &ip_dst);
242  if (ipp == NULL)
243  goto error;
244 
245  exp_list = IPPairGetStorageById(ipp, g_ippair_expectation_id);
246  if (exp_list) {
247  CIRCLEQ_INSERT_HEAD(&exp_list->list, exp, entries);
248  /* In case there is already EXPECTATION_MAX_LEVEL expectations waiting to be fullfill,
249  * we remove the older expectation to limit the total number of expectations */
250  if (exp_list->length >= EXPECTATION_MAX_LEVEL) {
251  Expectation *last_exp = CIRCLEQ_LAST(&exp_list->list);
252  CIRCLEQ_REMOVE(&exp_list->list, last_exp, entries);
253  AppLayerFreeExpectation(last_exp);
254  /* We keep the same amount of expectation so we fully release
255  * the IP pair */
257  IPPairRelease(ipp);
258  return 0;
259  }
260  } else {
261  exp_list = SCCalloc(1, sizeof(*exp_list));
262  if (exp_list == NULL)
263  goto error;
264  exp_list->length = 0;
265  CIRCLEQ_INIT(&exp_list->list);
266  CIRCLEQ_INSERT_HEAD(&exp_list->list, exp, entries);
267  IPPairSetStorageById(ipp, g_ippair_expectation_id, exp_list);
268  }
269 
270  exp_list->length += 1;
271  SC_ATOMIC_ADD(expectation_count, 1);
273  /* As we are creating the expectation, we release lock on IPPair without
274  * setting the ref count to 0. This way the IPPair will be kept till
275  * cleanup */
276  IPPairUnlock(ipp);
277  return 0;
278 
279 error:
280  SCFree(exp);
281  return -1;
282 }
283 
284 /**
285  * Return Flow storage identifier corresponding to expectation data
286  *
287  * \return expectation data identifier
288  */
290 {
291  return g_flow_expectation_id;
292 }
293 
294 /**
295  * Function doing a lookup in expectation list and updating Flow if needed.
296  *
297  * This function lookup for a existing expectation that could match the Flow.
298  * If found and if the expectation contains data it store the data in the
299  * expectation storage of the Flow.
300  *
301  * \return an AppProto value if found
302  * \return ALPROTO_UNKNOWN if not found
303  */
305 {
306  AppProto alproto = ALPROTO_UNKNOWN;
307  IPPair *ipp = NULL;
308  Expectation *lexp = NULL;
309  Expectation *exp = NULL;
310 
311  int x = SC_ATOMIC_GET(expectation_count);
312  if (x == 0) {
313  return ALPROTO_UNKNOWN;
314  }
315 
316  /* Call will take reference of the ip pair in 'ipp' */
317  ExpectationList *exp_list = AppLayerExpectationLookup(f, &ipp);
318  if (exp_list == NULL)
319  goto out;
320 
321  time_t ctime = f->lastts.tv_sec;
322 
323  CIRCLEQ_FOREACH_SAFE(exp, &exp_list->list, entries, lexp) {
324  if ((exp->direction & flags) && ((exp->sp == 0) || (exp->sp == f->sp)) &&
325  ((exp->dp == 0) || (exp->dp == f->dp))) {
326  alproto = exp->alproto;
327  f->alproto_ts = alproto;
328  f->alproto_tc = alproto;
329  void *fdata = FlowGetStorageById(f, g_flow_expectation_id);
330  if (fdata) {
331  /* We already have an expectation so let's clean this one */
332  ExpectationDataFree(exp->data);
333  } else {
334  /* Transfer ownership of Expectation data to the Flow */
335  if (FlowSetStorageById(f, g_flow_expectation_id, exp->data) != 0) {
336  SCLogDebug("Unable to set flow storage");
337  }
338  }
339  exp->data = NULL;
340  exp_list = AppLayerExpectationRemove(ipp, exp_list, exp);
341  if (exp_list == NULL)
342  goto out;
343  continue;
344  }
345  /* Cleaning remove old entries */
346  if (ctime > exp->ts.tv_sec + EXPECTATION_TIMEOUT) {
347  exp_list = AppLayerExpectationRemove(ipp, exp_list, exp);
348  if (exp_list == NULL)
349  goto out;
350  continue;
351  }
352  }
353 
354 out:
355  if (ipp)
356  IPPairRelease(ipp);
357  return alproto;
358 }
359 
361 {
362  IPPair *ipp = NULL;
363  Expectation *exp = NULL;
364  Expectation *pexp = NULL;
365 
366  int x = SC_ATOMIC_GET(expectation_count);
367  if (x == 0) {
368  return;
369  }
370 
371  /* Call will take reference of the ip pair in 'ipp' */
372  ExpectationList *exp_list = AppLayerExpectationLookup(f, &ipp);
373  if (exp_list == NULL)
374  goto out;
375 
376  CIRCLEQ_FOREACH_SAFE(exp, &exp_list->list, entries, pexp) {
377  /* Cleaning remove old entries */
378  if (exp->orig_f == (void *)f) {
379  exp_list = AppLayerExpectationRemove(ipp, exp_list, exp);
380  if (exp_list == NULL)
381  goto out;
382  }
383  }
384 
385 out:
386  if (ipp)
387  IPPairRelease(ipp);
388  return;
389 }
390 
391 /**
392  * @}
393  */
FlowStorageId
Definition: flow-storage.h:32
EXPECTATION_TIMEOUT
#define EXPECTATION_TIMEOUT
Definition: app-layer-expectation.c:71
Expectation_::alproto
AppProto alproto
Definition: app-layer-expectation.c:78
FLOW_HAS_EXPECTATION
#define FLOW_HAS_EXPECTATION
Definition: flow.h:112
CIRCLEQ_FOREACH_SAFE
#define CIRCLEQ_FOREACH_SAFE(var, head, field, tvar)
Definition: queue.h:553
ExpectationGetCounter
uint64_t ExpectationGetCounter(void)
Definition: app-layer-expectation.c:141
FLOW_IS_IPV6
#define FLOW_IS_IPV6(f)
Definition: flow.h:157
Expectation_::data
void * data
Definition: app-layer-expectation.c:82
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:315
AppLayerExpectationSetup
void AppLayerExpectationSetup(void)
Definition: app-layer-expectation.c:147
IPPairRelease
void IPPairRelease(IPPair *h)
Definition: ippair.c:518
CIRCLEQ_HEAD
#define CIRCLEQ_HEAD(name, type)
Definition: queue.h:439
Expectation_::orig_f
void * orig_f
Definition: app-layer-expectation.c:81
Expectation
struct Expectation_ Expectation
AppLayerExpectationGetFlowId
FlowStorageId AppLayerExpectationGetFlowId(void)
Definition: app-layer-expectation.c:289
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
ExpectationData_::DFree
void(* DFree)(void *)
Definition: app-layer-expectation.c:89
AppProto
uint16_t AppProto
Definition: app-layer-protos.h:77
Flow_
Flow data structure.
Definition: flow.h:350
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:333
Expectation_::ts
struct timeval ts
Definition: app-layer-expectation.c:75
Address_
Definition: decode.h:118
ExpectationData
struct ExpectationData_ ExpectationData
ExpectationData_
Definition: app-layer-expectation.c:86
Flow_::dp
Port dp
Definition: flow.h:366
AppLayerExpectationClean
void AppLayerExpectationClean(Flow *f)
Definition: app-layer-expectation.c:360
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint32_t, expectation_count)
FLOW_COPY_IPV6_ADDR_TO_PACKET
#define FLOW_COPY_IPV6_ADDR_TO_PACKET(fa, pa)
Definition: flow.h:170
Flow_::dst
FlowAddress dst
Definition: flow.h:353
Expectation_::sp
Port sp
Definition: app-layer-expectation.c:76
app-layer-expectation.h
AppLayerExpectationHandle
AppProto AppLayerExpectationHandle(Flow *f, uint8_t flags)
Definition: app-layer-expectation.c:304
ExpectationList
struct ExpectationList_ ExpectationList
util-print.h
FlowSetStorageById
int FlowSetStorageById(Flow *f, FlowStorageId id, void *ptr)
Definition: flow-storage.c:44
FlowStorageRegister
FlowStorageId FlowStorageRegister(const char *name, const unsigned int size, void *(*Alloc)(unsigned int), void(*Free)(void *))
Definition: flow-storage.c:65
IPPairGetIPPairFromHash
IPPair * IPPairGetIPPairFromHash(Address *a, Address *b)
Definition: ippair.c:542
FLOW_IS_IPV4
#define FLOW_IS_IPV4(f)
Definition: flow.h:155
IPPairUnlock
void IPPairUnlock(IPPair *h)
Definition: ippair.c:529
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:342
EXPECTATION_MAX_LEVEL
#define EXPECTATION_MAX_LEVEL
Definition: app-layer-expectation.c:72
AppLayerExpectationCreate
int AppLayerExpectationCreate(Flow *f, int direction, Port src, Port dst, AppProto alproto, void *data)
Definition: app-layer-expectation.c:220
CIRCLEQ_ENTRY
#define CIRCLEQ_ENTRY(type)
Definition: queue.h:448
Port
uint16_t Port
Definition: decode.h:242
queue.h
FlowGetStorageById
void * FlowGetStorageById(Flow *f, FlowStorageId id)
Definition: flow-storage.c:39
IPPairStorageRegister
int IPPairStorageRegister(const char *name, const unsigned int size, void *(*Alloc)(unsigned int), void(*Free)(void *))
Definition: ippair-storage.c:61
ExpectationList_
Definition: app-layer-expectation.c:92
Flow_::src
FlowAddress src
Definition: flow.h:353
Flow_::lastts
struct timeval lastts
Definition: flow.h:411
IPPairGetStorageById
void * IPPairGetStorageById(IPPair *h, int id)
Definition: ippair-storage.c:35
length
uint16_t length
Definition: app-layer-modbus.c:2
flow-storage.h
flags
uint8_t flags
Definition: decode-gre.h:0
suricata-common.h
IPPair_
Definition: ippair.h:58
CIRCLEQ_REMOVE
#define CIRCLEQ_REMOVE(head, elm, field)
Definition: queue.h:523
CIRCLEQ_INSERT_HEAD
#define CIRCLEQ_INSERT_HEAD(head, elm, field)
Definition: queue.h:503
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:458
CIRCLEQ_INIT
#define CIRCLEQ_INIT(head)
Definition: queue.h:478
Flow_::flags
uint32_t flags
Definition: flow.h:428
src
uint16_t src
Definition: app-layer-dnp3.h:5
FlowStorageId::id
int id
Definition: flow-storage.h:33
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
IPPairLookupIPPairFromHash
IPPair * IPPairLookupIPPairFromHash(Address *a, Address *b)
look up a ippair in the hash
Definition: ippair.c:641
Expectation_::dp
Port dp
Definition: app-layer-expectation.c:77
Flow_::sp
Port sp
Definition: flow.h:355
dst
uint16_t dst
Definition: app-layer-dnp3.h:4
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:376
IPPairSetStorageById
int IPPairSetStorageById(IPPair *h, int id, void *ptr)
Definition: ippair-storage.c:40
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:459
ippair-storage.h
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
Expectation_
Definition: app-layer-expectation.c:74
debug.h
Expectation_::direction
int direction
Definition: app-layer-expectation.c:79
FLOW_COPY_IPV4_ADDR_TO_PACKET
#define FLOW_COPY_IPV4_ADDR_TO_PACKET(fa, pa)
Definition: flow.h:165
CIRCLEQ_LAST
#define CIRCLEQ_LAST(head)
Definition: queue.h:458