suricata
app-layer-expectation.c
Go to the documentation of this file.
1 /* Copyright (C) 2017-2021 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \defgroup applayerexpectation Application Layer Expectation
20  *
21  * Handling of dynamic parallel connection for application layer similar
22  * to FTP.
23  *
24  * @{
25  *
26  * Some protocols like FTP create dynamic parallel flow (called expectation). In
27  * order to assign a application layer protocol to these expectation, Suricata
28  * needs to parse message of the initial protocol and create and maintain a list
29  * of expected flow.
30  *
31  * Application layers must use the here described API to implement this mechanism.
32  *
33  * When parsing a application layer message describing a parallel flow, the
34  * application layer can call AppLayerExpectationCreate() to declare an
35  * expectation. By doing that the next flow coming with corresponding IP parameters
36  * will be assigned the specified application layer. The resulting Flow will
37  * also have a Flow storage set that can be retrieved at index
38  * AppLayerExpectationGetDataId():
39  *
40  * ```
41  * data = (char *)FlowGetStorageById(f, AppLayerExpectationGetFlowId());
42  * ```
43  * This storage can be used to store information that are only available in the
44  * parent connection and could be useful in the parent connection. For instance
45  * this is used by the FTP protocol to propagate information such as file name
46  * and ftp operation to the FTP data connection.
47  */
48 
49 /**
50  * \file
51  *
52  * \author Eric Leblond <eric@regit.org>
53  */
54 
55 #include "queue.h"
56 #include "suricata-common.h"
57 
58 #include "ippair-storage.h"
59 #include "flow-storage.h"
60 
61 #include "app-layer-expectation.h"
62 
63 #include "util-print.h"
64 
65 static IPPairStorageId g_ippair_expectation_id = { .id = -1 };
66 static FlowStorageId g_flow_expectation_id = { .id = -1 };
67 
68 SC_ATOMIC_DECLARE(uint32_t, expectation_count);
69 
70 #define EXPECTATION_TIMEOUT 30
71 #define EXPECTATION_MAX_LEVEL 10
72 
73 typedef struct Expectation_ {
74  struct timeval ts;
78  int direction;
79  /* use pointer to Flow as identifier of the Flow the expectation is linked to */
80  void *orig_f;
81  void *data;
82  CIRCLEQ_ENTRY(Expectation_) entries;
84 
85 typedef struct ExpectationData_ {
86  /** Start of Expectation Data structure must be a pointer
87  * to free function. Set to NULL to use SCFree() */
88  void (*DFree)(void *);
90 
91 typedef struct ExpectationList_ {
92  CIRCLEQ_HEAD(EList, Expectation_) list;
93  uint8_t length;
95 
96 static void ExpectationDataFree(void *e)
97 {
98  SCLogDebug("Free expectation data");
99  ExpectationData *ed = (ExpectationData *) e;
100  if (ed->DFree) {
101  ed->DFree(e);
102  } else {
103  SCFree(e);
104  }
105 }
106 
107 /**
108  * Free expectation
109  */
110 static void AppLayerFreeExpectation(Expectation *exp)
111 {
112  if (exp->data) {
113  ExpectationData *expdata = (ExpectationData *)exp->data;
114  if (expdata->DFree) {
115  expdata->DFree(exp->data);
116  } else {
117  SCFree(exp->data);
118  }
119  }
120  SCFree(exp);
121 }
122 
123 static void ExpectationListFree(void *el)
124 {
125  ExpectationList *exp_list = (ExpectationList *)el;
126  if (exp_list == NULL)
127  return;
128 
129  if (exp_list->length > 0) {
130  Expectation *exp = NULL, *pexp = NULL;
131  CIRCLEQ_FOREACH_SAFE(exp, &exp_list->list, entries, pexp) {
132  CIRCLEQ_REMOVE(&exp_list->list, exp, entries);
133  exp_list->length--;
134  AppLayerFreeExpectation(exp);
135  }
136  }
137  SCFree(exp_list);
138 }
139 
140 uint64_t ExpectationGetCounter(void)
141 {
142  uint64_t x = SC_ATOMIC_GET(expectation_count);
143  return x;
144 }
145 
147 {
148  g_ippair_expectation_id =
149  IPPairStorageRegister("expectation", sizeof(void *), NULL, ExpectationListFree);
150  g_flow_expectation_id =
151  FlowStorageRegister("expectation", sizeof(void *), NULL, ExpectationDataFree);
152  SC_ATOMIC_INIT(expectation_count);
153 }
154 
155 static inline int GetFlowAddresses(Flow *f, Address *ip_src, Address *ip_dst)
156 {
157  memset(ip_src, 0, sizeof(*ip_src));
158  memset(ip_dst, 0, sizeof(*ip_dst));
159  if (FLOW_IS_IPV4(f)) {
160  FLOW_COPY_IPV4_ADDR_TO_PACKET(&f->src, ip_src);
161  FLOW_COPY_IPV4_ADDR_TO_PACKET(&f->dst, ip_dst);
162  } else if (FLOW_IS_IPV6(f)) {
163  FLOW_COPY_IPV6_ADDR_TO_PACKET(&f->src, ip_src);
164  FLOW_COPY_IPV6_ADDR_TO_PACKET(&f->dst, ip_dst);
165  } else {
166  return -1;
167  }
168  return 0;
169 }
170 
171 static ExpectationList *AppLayerExpectationLookup(Flow *f, IPPair **ipp)
172 {
173  Address ip_src, ip_dst;
174  if (GetFlowAddresses(f, &ip_src, &ip_dst) == -1)
175  return NULL;
176  *ipp = IPPairLookupIPPairFromHash(&ip_src, &ip_dst);
177  if (*ipp == NULL) {
178  return NULL;
179  }
180 
181  return IPPairGetStorageById(*ipp, g_ippair_expectation_id);
182 }
183 
184 
185 static ExpectationList *AppLayerExpectationRemove(IPPair *ipp,
186  ExpectationList *exp_list,
187  Expectation *exp)
188 {
189  CIRCLEQ_REMOVE(&exp_list->list, exp, entries);
190  AppLayerFreeExpectation(exp);
191  SC_ATOMIC_SUB(expectation_count, 1);
192  exp_list->length--;
193  if (exp_list->length == 0) {
194  IPPairSetStorageById(ipp, g_ippair_expectation_id, NULL);
195  ExpectationListFree(exp_list);
196  exp_list = NULL;
197  }
198  return exp_list;
199 }
200 
201 /**
202  * Create an entry in expectation list
203  *
204  * Create a expectation from an existing Flow. Currently, only Flow between
205  * the two original IP addresses are supported. In case of success, the
206  * ownership of the data pointer is taken. In case of error, the pointer
207  * to data has to be freed by the caller.
208  *
209  * \param f a pointer to the original Flow
210  * \param direction the direction of the data in the expectation flow
211  * \param src source port of the expected flow, use 0 for any
212  * \param dst destination port of the expected flow, use 0 for any
213  * \param alproto the protocol that need to be set on the expected flow
214  * \param data pointer to data that will be attached to the expected flow
215  *
216  * \return -1 if error
217  * \return 0 if success
218  */
219 int AppLayerExpectationCreate(Flow *f, int direction, Port src, Port dst,
220  AppProto alproto, void *data)
221 {
222  ExpectationList *exp_list = NULL;
223  IPPair *ipp;
224  Address ip_src, ip_dst;
225 
226  Expectation *exp = SCCalloc(1, sizeof(*exp));
227  if (exp == NULL)
228  return -1;
229 
230  exp->sp = src;
231  exp->dp = dst;
232  exp->alproto = alproto;
233  exp->ts = f->lastts;
234  exp->orig_f = (void *)f;
235  exp->data = data;
236  exp->direction = direction;
237 
238  if (GetFlowAddresses(f, &ip_src, &ip_dst) == -1)
239  goto error;
240  ipp = IPPairGetIPPairFromHash(&ip_src, &ip_dst);
241  if (ipp == NULL)
242  goto error;
243 
244  exp_list = IPPairGetStorageById(ipp, g_ippair_expectation_id);
245  if (exp_list) {
246  CIRCLEQ_INSERT_HEAD(&exp_list->list, exp, entries);
247  /* In case there is already EXPECTATION_MAX_LEVEL expectations waiting to be fullfill,
248  * we remove the older expectation to limit the total number of expectations */
249  if (exp_list->length >= EXPECTATION_MAX_LEVEL) {
250  Expectation *last_exp = CIRCLEQ_LAST(&exp_list->list);
251  CIRCLEQ_REMOVE(&exp_list->list, last_exp, entries);
252  AppLayerFreeExpectation(last_exp);
253  /* We keep the same amount of expectation so we fully release
254  * the IP pair */
256  IPPairRelease(ipp);
257  return 0;
258  }
259  } else {
260  exp_list = SCCalloc(1, sizeof(*exp_list));
261  if (exp_list == NULL)
262  goto error;
263  exp_list->length = 0;
264  CIRCLEQ_INIT(&exp_list->list);
265  CIRCLEQ_INSERT_HEAD(&exp_list->list, exp, entries);
266  IPPairSetStorageById(ipp, g_ippair_expectation_id, exp_list);
267  }
268 
269  exp_list->length += 1;
270  SC_ATOMIC_ADD(expectation_count, 1);
272  /* As we are creating the expectation, we release lock on IPPair without
273  * setting the ref count to 0. This way the IPPair will be kept till
274  * cleanup */
275  IPPairUnlock(ipp);
276  return 0;
277 
278 error:
279  SCFree(exp);
280  return -1;
281 }
282 
283 /**
284  * Return Flow storage identifier corresponding to expectation data
285  *
286  * \return expectation data identifier
287  */
289 {
290  return g_flow_expectation_id;
291 }
292 
293 /**
294  * Function doing a lookup in expectation list and updating Flow if needed.
295  *
296  * This function lookup for a existing expectation that could match the Flow.
297  * If found and if the expectation contains data it store the data in the
298  * expectation storage of the Flow.
299  *
300  * \return an AppProto value if found
301  * \return ALPROTO_UNKNOWN if not found
302  */
304 {
305  AppProto alproto = ALPROTO_UNKNOWN;
306  IPPair *ipp = NULL;
307  Expectation *lexp = NULL;
308  Expectation *exp = NULL;
309 
310  int x = SC_ATOMIC_GET(expectation_count);
311  if (x == 0) {
312  return ALPROTO_UNKNOWN;
313  }
314 
315  /* Call will take reference of the ip pair in 'ipp' */
316  ExpectationList *exp_list = AppLayerExpectationLookup(f, &ipp);
317  if (exp_list == NULL)
318  goto out;
319 
320  time_t ctime = f->lastts.tv_sec;
321 
322  CIRCLEQ_FOREACH_SAFE(exp, &exp_list->list, entries, lexp) {
323  if ((exp->direction & flags) && ((exp->sp == 0) || (exp->sp == f->sp)) &&
324  ((exp->dp == 0) || (exp->dp == f->dp))) {
325  alproto = exp->alproto;
326  if (f->alproto_ts == ALPROTO_UNKNOWN) {
327  f->alproto_ts = alproto;
328  }
329  if (f->alproto_tc == ALPROTO_UNKNOWN) {
330  f->alproto_tc = alproto;
331  }
332  void *fdata = FlowGetStorageById(f, g_flow_expectation_id);
333  if (fdata) {
334  /* We already have an expectation so let's clean this one */
335  ExpectationDataFree(exp->data);
336  } else {
337  /* Transfer ownership of Expectation data to the Flow */
338  if (FlowSetStorageById(f, g_flow_expectation_id, exp->data) != 0) {
339  SCLogDebug("Unable to set flow storage");
340  }
341  }
342  exp->data = NULL;
343  exp_list = AppLayerExpectationRemove(ipp, exp_list, exp);
344  if (exp_list == NULL)
345  goto out;
346  continue;
347  }
348  /* Cleaning remove old entries */
349  if (ctime > exp->ts.tv_sec + EXPECTATION_TIMEOUT) {
350  exp_list = AppLayerExpectationRemove(ipp, exp_list, exp);
351  if (exp_list == NULL)
352  goto out;
353  continue;
354  }
355  }
356 
357 out:
358  if (ipp)
359  IPPairRelease(ipp);
360  return alproto;
361 }
362 
364 {
365  IPPair *ipp = NULL;
366  Expectation *exp = NULL;
367  Expectation *pexp = NULL;
368 
369  int x = SC_ATOMIC_GET(expectation_count);
370  if (x == 0) {
371  return;
372  }
373 
374  /* Call will take reference of the ip pair in 'ipp' */
375  ExpectationList *exp_list = AppLayerExpectationLookup(f, &ipp);
376  if (exp_list == NULL)
377  goto out;
378 
379  CIRCLEQ_FOREACH_SAFE(exp, &exp_list->list, entries, pexp) {
380  /* Cleaning remove old entries */
381  if (exp->orig_f == (void *)f) {
382  exp_list = AppLayerExpectationRemove(ipp, exp_list, exp);
383  if (exp_list == NULL)
384  goto out;
385  }
386  }
387 
388 out:
389  if (ipp)
390  IPPairRelease(ipp);
391  return;
392 }
393 
394 /**
395  * @}
396  */
FlowStorageId
Definition: flow-storage.h:31
IPPairStorageRegister
IPPairStorageId IPPairStorageRegister(const char *name, const unsigned int size, void *(*Alloc)(unsigned int), void(*Free)(void *))
Definition: ippair-storage.c:61
EXPECTATION_TIMEOUT
#define EXPECTATION_TIMEOUT
Definition: app-layer-expectation.c:70
Expectation_::alproto
AppProto alproto
Definition: app-layer-expectation.c:77
FLOW_HAS_EXPECTATION
#define FLOW_HAS_EXPECTATION
Definition: flow.h:110
CIRCLEQ_FOREACH_SAFE
#define CIRCLEQ_FOREACH_SAFE(var, head, field, tvar)
Definition: queue.h:106
ExpectationGetCounter
uint64_t ExpectationGetCounter(void)
Definition: app-layer-expectation.c:140
FLOW_IS_IPV6
#define FLOW_IS_IPV6(f)
Definition: flow.h:161
Expectation_::data
void * data
Definition: app-layer-expectation.c:81
SC_ATOMIC_INIT
#define SC_ATOMIC_INIT(name)
wrapper for initializing an atomic variable.
Definition: util-atomic.h:315
AppLayerExpectationSetup
void AppLayerExpectationSetup(void)
Definition: app-layer-expectation.c:146
IPPairRelease
void IPPairRelease(IPPair *h)
Definition: ippair.c:521
CIRCLEQ_HEAD
#define CIRCLEQ_HEAD(name, type)
Definition: queue.h:76
Expectation_::orig_f
void * orig_f
Definition: app-layer-expectation.c:80
Expectation
struct Expectation_ Expectation
AppLayerExpectationGetFlowId
FlowStorageId AppLayerExpectationGetFlowId(void)
Definition: app-layer-expectation.c:288
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:296
ExpectationData_::DFree
void(* DFree)(void *)
Definition: app-layer-expectation.c:88
AppProto
uint16_t AppProto
Definition: app-layer-protos.h:81
Flow_
Flow data structure.
Definition: flow.h:356
SC_ATOMIC_ADD
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
Definition: util-atomic.h:333
Expectation_::ts
struct timeval ts
Definition: app-layer-expectation.c:74
Address_
Definition: decode.h:111
ExpectationData
struct ExpectationData_ ExpectationData
ExpectationData_
Definition: app-layer-expectation.c:85
IPPairSetStorageById
int IPPairSetStorageById(IPPair *h, IPPairStorageId id, void *ptr)
Definition: ippair-storage.c:40
Flow_::dp
Port dp
Definition: flow.h:372
IPPairStorageId::id
int id
Definition: ippair-storage.h:32
AppLayerExpectationClean
void AppLayerExpectationClean(Flow *f)
Definition: app-layer-expectation.c:363
SC_ATOMIC_DECLARE
SC_ATOMIC_DECLARE(uint32_t, expectation_count)
FLOW_COPY_IPV6_ADDR_TO_PACKET
#define FLOW_COPY_IPV6_ADDR_TO_PACKET(fa, pa)
Definition: flow.h:174
Flow_::dst
FlowAddress dst
Definition: flow.h:359
Expectation_::sp
Port sp
Definition: app-layer-expectation.c:75
app-layer-expectation.h
AppLayerExpectationHandle
AppProto AppLayerExpectationHandle(Flow *f, uint8_t flags)
Definition: app-layer-expectation.c:303
ExpectationList
struct ExpectationList_ ExpectationList
util-print.h
FlowSetStorageById
int FlowSetStorageById(Flow *f, FlowStorageId id, void *ptr)
Definition: flow-storage.c:45
FlowStorageRegister
FlowStorageId FlowStorageRegister(const char *name, const unsigned int size, void *(*Alloc)(unsigned int), void(*Free)(void *))
Definition: flow-storage.c:66
IPPairGetIPPairFromHash
IPPair * IPPairGetIPPairFromHash(Address *a, Address *b)
Definition: ippair.c:545
FLOW_IS_IPV4
#define FLOW_IS_IPV4(f)
Definition: flow.h:159
IPPairUnlock
void IPPairUnlock(IPPair *h)
Definition: ippair.c:532
SC_ATOMIC_SUB
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
Definition: util-atomic.h:342
EXPECTATION_MAX_LEVEL
#define EXPECTATION_MAX_LEVEL
Definition: app-layer-expectation.c:71
AppLayerExpectationCreate
int AppLayerExpectationCreate(Flow *f, int direction, Port src, Port dst, AppProto alproto, void *data)
Definition: app-layer-expectation.c:219
CIRCLEQ_ENTRY
#define CIRCLEQ_ENTRY(type)
Definition: queue.h:85
Port
uint16_t Port
Definition: decode.h:226
queue.h
FlowGetStorageById
void * FlowGetStorageById(const Flow *f, FlowStorageId id)
Definition: flow-storage.c:40
ExpectationList_
Definition: app-layer-expectation.c:91
Flow_::src
FlowAddress src
Definition: flow.h:359
Flow_::lastts
struct timeval lastts
Definition: flow.h:417
flow-storage.h
flags
uint8_t flags
Definition: decode-gre.h:0
suricata-common.h
IPPair_
Definition: ippair.h:58
CIRCLEQ_REMOVE
#define CIRCLEQ_REMOVE(head, elm, field)
Definition: queue.h:171
CIRCLEQ_INSERT_HEAD
#define CIRCLEQ_INSERT_HEAD(head, elm, field)
Definition: queue.h:151
IPPairGetStorageById
void * IPPairGetStorageById(IPPair *h, IPPairStorageId id)
Definition: ippair-storage.c:35
SCFree
#define SCFree(p)
Definition: util-mem.h:61
Flow_::alproto_ts
AppProto alproto_ts
Definition: flow.h:464
CIRCLEQ_INIT
#define CIRCLEQ_INIT(head)
Definition: queue.h:126
Flow_::flags
uint32_t flags
Definition: flow.h:434
src
uint16_t src
Definition: app-layer-dnp3.h:5
FlowStorageId::id
int id
Definition: flow-storage.h:32
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
IPPairLookupIPPairFromHash
IPPair * IPPairLookupIPPairFromHash(Address *a, Address *b)
look up a ippair in the hash
Definition: ippair.c:644
Expectation_::dp
Port dp
Definition: app-layer-expectation.c:76
Flow_::sp
Port sp
Definition: flow.h:361
dst
uint16_t dst
Definition: app-layer-dnp3.h:4
SC_ATOMIC_GET
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
Definition: util-atomic.h:376
Flow_::alproto_tc
AppProto alproto_tc
Definition: flow.h:465
ippair-storage.h
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
IPPairStorageId
Definition: ippair-storage.h:31
Expectation_
Definition: app-layer-expectation.c:73
Expectation_::direction
int direction
Definition: app-layer-expectation.c:78
FLOW_COPY_IPV4_ADDR_TO_PACKET
#define FLOW_COPY_IPV4_ADDR_TO_PACKET(fa, pa)
Definition: flow.h:169
CIRCLEQ_LAST
#define CIRCLEQ_LAST(head)
Definition: queue.h:95