suricata
output-lua.c
Go to the documentation of this file.
1 /* Copyright (C) 2014 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  */
24 
25 #include "suricata-common.h"
26 #include "debug.h"
27 #include "pkt-var.h"
28 #include "conf.h"
29 
30 #include "threads.h"
31 #include "threadvars.h"
32 #include "tm-threads.h"
33 
34 #include "util-print.h"
35 #include "util-unittest.h"
36 
37 #include "util-debug.h"
38 
39 #include "output.h"
40 #include "app-layer-htp.h"
41 #include "app-layer.h"
42 #include "app-layer-ssl.h"
43 #include "app-layer-ssh.h"
44 #include "app-layer-parser.h"
45 #include "util-privs.h"
46 #include "util-buffer.h"
47 #include "util-proto-name.h"
48 #include "util-logopenfile.h"
49 #include "util-time.h"
50 
51 #include "output-lua.h"
52 
53 #ifdef HAVE_LUA
54 
55 #include <lua.h>
56 #include <lualib.h>
57 #include <lauxlib.h>
58 
59 #include "util-lua.h"
60 #include "util-lua-common.h"
61 #include "util-lua-http.h"
62 #include "util-lua-dns.h"
63 #include "util-lua-ja3.h"
64 #include "util-lua-tls.h"
65 #include "util-lua-ssh.h"
66 #include "util-lua-smtp.h"
67 
68 #define MODULE_NAME "LuaLog"
69 
70 /** \brief structure containing global config
71  * The OutputLuaLogInitSub which is run per script
72  * can access this to get global config info through
73  * it's parent_ctx->data ptr.
74  */
75 typedef struct LogLuaMasterCtx_ {
76  char path[PATH_MAX]; /**< contains script-dir */
77 } LogLuaMasterCtx;
78 
79 typedef struct LogLuaCtx_ {
80  SCMutex m;
81  lua_State *luastate;
82  int deinit_once;
83 } LogLuaCtx;
84 
85 typedef struct LogLuaThreadCtx_ {
86  LogLuaCtx *lua_ctx;
87 } LogLuaThreadCtx;
88 
89 static TmEcode LuaLogThreadInit(ThreadVars *t, const void *initdata, void **data);
90 static TmEcode LuaLogThreadDeinit(ThreadVars *t, void *data);
91 
92 /** \internal
93  * \brief TX logger for lua scripts
94  *
95  * A single call to this function will run one script on a single
96  * transaction.
97  *
98  * NOTE: The flow (f) also referenced by p->flow is locked.
99  */
100 static int LuaTxLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *alstate, void *txptr, uint64_t tx_id)
101 {
102  SCEnter();
103 
104  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
105 
106  SCMutexLock(&td->lua_ctx->m);
107 
108  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
109  LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
110  LuaStateSetTX(td->lua_ctx->luastate, txptr);
111  LuaStateSetFlow(td->lua_ctx->luastate, f);
112 
113  /* prepare data to pass to script */
114  lua_getglobal(td->lua_ctx->luastate, "log");
115  lua_newtable(td->lua_ctx->luastate);
116  LuaPushTableKeyValueInt(td->lua_ctx->luastate, "tx_id", (int)(tx_id));
117 
118  int retval = lua_pcall(td->lua_ctx->luastate, 1, 0, 0);
119  if (retval != 0) {
120  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
121  }
122 
123  SCMutexUnlock(&td->lua_ctx->m);
124  SCReturnInt(0);
125 }
126 
127 /** \internal
128  * \brief Streaming logger for lua scripts
129  *
130  * Hooks into the Streaming Logger API. Gets called for each chunk of new
131  * streaming data.
132  */
133 static int LuaStreamingLogger(ThreadVars *tv, void *thread_data, const Flow *f,
134  const uint8_t *data, uint32_t data_len, uint64_t tx_id, uint8_t flags)
135 {
136  SCEnter();
137 
138  void *txptr = NULL;
139  LuaStreamingBuffer b = { data, data_len, flags };
140 
141  SCLogDebug("flags %02x", flags);
142 
143  if (flags & OUTPUT_STREAMING_FLAG_TRANSACTION) {
144  if (f && f->alstate)
145  txptr = AppLayerParserGetTx(f->proto, f->alproto, f->alstate, tx_id);
146  }
147 
148  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
149 
150  SCMutexLock(&td->lua_ctx->m);
151 
152  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
153  if (flags & OUTPUT_STREAMING_FLAG_TRANSACTION)
154  LuaStateSetTX(td->lua_ctx->luastate, txptr);
155  LuaStateSetFlow(td->lua_ctx->luastate, (Flow *)f);
156  LuaStateSetStreamingBuffer(td->lua_ctx->luastate, &b);
157 
158  /* prepare data to pass to script */
159  lua_getglobal(td->lua_ctx->luastate, "log");
160  lua_newtable(td->lua_ctx->luastate);
161 
162  if (flags & OUTPUT_STREAMING_FLAG_TRANSACTION)
163  LuaPushTableKeyValueInt(td->lua_ctx->luastate, "tx_id", (int)(tx_id));
164 
165  int retval = lua_pcall(td->lua_ctx->luastate, 1, 0, 0);
166  if (retval != 0) {
167  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
168  }
169 
170  SCMutexUnlock(&td->lua_ctx->m);
171 
173 }
174 
175 /** \internal
176  * \brief Packet Logger for lua scripts, for alerts
177  *
178  * A single call to this function will run one script for a single
179  * packet. If it is called, it means that the registered condition
180  * function has returned TRUE.
181  *
182  * The script is called once for each alert stored in the packet.
183  *
184  * NOTE: p->flow is UNlocked
185  */
186 static int LuaPacketLoggerAlerts(ThreadVars *tv, void *thread_data, const Packet *p)
187 {
188  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
189 
190  char timebuf[64];
191  CreateTimeString(&p->ts, timebuf, sizeof(timebuf));
192 
193  if (!(PKT_IS_IPV4(p)) && !(PKT_IS_IPV6(p))) {
194  /* decoder event */
195  goto not_supported;
196  }
197 
198  char proto[16] = "";
199  if (SCProtoNameValid(IP_GET_IPPROTO(p)) == TRUE) {
200  strlcpy(proto, known_proto[IP_GET_IPPROTO(p)], sizeof(proto));
201  } else {
202  snprintf(proto, sizeof(proto), "PROTO:%03" PRIu32, IP_GET_IPPROTO(p));
203  }
204 
205  /* loop through alerts stored in the packet */
206  SCMutexLock(&td->lua_ctx->m);
207  uint16_t cnt;
208  for (cnt = 0; cnt < p->alerts.cnt; cnt++) {
209  const PacketAlert *pa = &p->alerts.alerts[cnt];
210  if (unlikely(pa->s == NULL)) {
211  continue;
212  }
213 
214  lua_getglobal(td->lua_ctx->luastate, "log");
215 
216  void *txptr = NULL;
217  if (p->flow && p->flow->alstate && (pa->flags & PACKET_ALERT_FLAG_TX))
218  txptr = AppLayerParserGetTx(p->proto, p->flow->alproto, p->flow->alstate, pa->tx_id);
219 
220  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
221  LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
222  LuaStateSetTX(td->lua_ctx->luastate, txptr);
223  LuaStateSetFlow(td->lua_ctx->luastate, p->flow);
224  LuaStateSetPacketAlert(td->lua_ctx->luastate, (PacketAlert *)pa);
225 
226  /* prepare data to pass to script */
227  //lua_newtable(td->lua_ctx->luastate);
228 
229  int retval = lua_pcall(td->lua_ctx->luastate, 0, 0, 0);
230  if (retval != 0) {
231  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
232  }
233  }
234  SCMutexUnlock(&td->lua_ctx->m);
235 not_supported:
236  SCReturnInt(0);
237 }
238 
239 static int LuaPacketConditionAlerts(ThreadVars *tv, const Packet *p)
240 {
241  if (p->alerts.cnt > 0)
242  return TRUE;
243  return FALSE;
244 }
245 
246 /** \internal
247  * \brief Packet Logger for lua scripts, for packets
248  *
249  * A single call to this function will run one script for a single
250  * packet. If it is called, it means that the registered condition
251  * function has returned TRUE.
252  *
253  * The script is called once for each packet.
254  *
255  * NOTE: p->flow is UNlocked
256  */
257 static int LuaPacketLogger(ThreadVars *tv, void *thread_data, const Packet *p)
258 {
259  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
260 
261  char timebuf[64];
262 
263  if ((!(PKT_IS_IPV4(p))) && (!(PKT_IS_IPV6(p)))) {
264  goto not_supported;
265  }
266 
267  CreateTimeString(&p->ts, timebuf, sizeof(timebuf));
268 
269  char proto[16] = "";
270  if (SCProtoNameValid(IP_GET_IPPROTO(p)) == TRUE) {
271  strlcpy(proto, known_proto[IP_GET_IPPROTO(p)], sizeof(proto));
272  } else {
273  snprintf(proto, sizeof(proto), "PROTO:%03" PRIu32, IP_GET_IPPROTO(p));
274  }
275 
276  /* loop through alerts stored in the packet */
277  SCMutexLock(&td->lua_ctx->m);
278  lua_getglobal(td->lua_ctx->luastate, "log");
279 
280  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
281  LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
282  LuaStateSetFlow(td->lua_ctx->luastate, p->flow);
283 
284  /* prepare data to pass to script */
285  lua_newtable(td->lua_ctx->luastate);
286 
287  int retval = lua_pcall(td->lua_ctx->luastate, 1, 0, 0);
288  if (retval != 0) {
289  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
290  }
291  SCMutexUnlock(&td->lua_ctx->m);
292 not_supported:
293  SCReturnInt(0);
294 }
295 
296 static int LuaPacketCondition(ThreadVars *tv, const Packet *p)
297 {
298  return TRUE;
299 }
300 
301 /** \internal
302  * \brief File API Logger function for Lua scripts
303  *
304  * Executes a script once for one file.
305  *
306  * NOTE p->flow is locked at this point
307  */
308 static int LuaFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff, uint8_t dir)
309 {
310  SCEnter();
311  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
312 
313  if ((!(PKT_IS_IPV4(p))) && (!(PKT_IS_IPV6(p))))
314  return 0;
315 
316  BUG_ON(ff->flags & FILE_LOGGED);
317 
318  SCLogDebug("ff %p", ff);
319 
320  SCMutexLock(&td->lua_ctx->m);
321 
322  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
323  LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
324  if (p->flow && p->flow->alstate) {
325  void *txptr = AppLayerParserGetTx(p->proto, p->flow->alproto, p->flow->alstate, ff->txid);
326  if (txptr) {
327  LuaStateSetTX(td->lua_ctx->luastate, txptr);
328  }
329  }
330  LuaStateSetFlow(td->lua_ctx->luastate, p->flow);
331  LuaStateSetFile(td->lua_ctx->luastate, (File *)ff);
332 
333  /* get the lua function to call */
334  lua_getglobal(td->lua_ctx->luastate, "log");
335 
336  int retval = lua_pcall(td->lua_ctx->luastate, 0, 0, 0);
337  if (retval != 0) {
338  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
339  }
340  SCMutexUnlock(&td->lua_ctx->m);
341  return 0;
342 }
343 
344 /** \internal
345  * \brief Flow API Logger function for Lua scripts
346  *
347  * Executes a script once for one flow
348  *
349  * Note: flow 'f' is locked
350  */
351 static int LuaFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
352 {
353  SCEnter();
354  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
355 
356  SCLogDebug("f %p", f);
357 
358  SCMutexLock(&td->lua_ctx->m);
359 
360  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
361  LuaStateSetFlow(td->lua_ctx->luastate, f);
362 
363  /* get the lua function to call */
364  lua_getglobal(td->lua_ctx->luastate, "log");
365 
366  int retval = lua_pcall(td->lua_ctx->luastate, 0, 0, 0);
367  if (retval != 0) {
368  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
369  }
370  SCMutexUnlock(&td->lua_ctx->m);
371  return 0;
372 }
373 
374 
375 
376 static int LuaStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable *st)
377 {
378  SCEnter();
379  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
380 
381  SCMutexLock(&td->lua_ctx->m);
382 
383  lua_State *luastate = td->lua_ctx->luastate;
384  /* get the lua function to call */
385  lua_getglobal(td->lua_ctx->luastate, "log");
386 
387  /* create lua array, which is really just a table. The key is an int (1-x),
388  * the value another table with named fields: name, tm_name, value, pvalue.
389  * { 1, { name=<name>, tmname=<tm_name>, value=<value>, pvalue=<pvalue>}}
390  * { 2, { name=<name>, tmname=<tm_name>, value=<value>, pvalue=<pvalue>}}
391  * etc
392  */
393  lua_newtable(luastate);
394  uint32_t u = 0;
395  for (; u < st->nstats; u++) {
396  lua_pushinteger(luastate, u + 1);
397 
398  lua_newtable(luastate);
399 
400  lua_pushstring(luastate, "name");
401  lua_pushstring(luastate, st->stats[u].name);
402  lua_settable(luastate, -3);
403 
404  lua_pushstring(luastate, "tmname");
405  lua_pushstring(luastate, st->stats[u].tm_name);
406  lua_settable(luastate, -3);
407 
408  lua_pushstring(luastate, "value");
409  lua_pushinteger(luastate, st->stats[u].value);
410  lua_settable(luastate, -3);
411 
412  lua_pushstring(luastate, "pvalue");
413  lua_pushinteger(luastate, st->stats[u].pvalue);
414  lua_settable(luastate, -3);
415 
416  lua_settable(luastate, -3);
417  }
418 
419  int retval = lua_pcall(td->lua_ctx->luastate, 1, 0, 0);
420  if (retval != 0) {
421  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
422  }
423  SCMutexUnlock(&td->lua_ctx->m);
424  return 0;
425 
426 }
427 
428 typedef struct LogLuaScriptOptions_ {
429  AppProto alproto;
430  int packet;
431  int alerts;
432  int file;
433  int streaming;
434  int tcp_data;
435  int http_body;
436  int flow;
437  int stats;
438 } LogLuaScriptOptions;
439 
440 /** \brief load and evaluate the script
441  *
442  * This function parses the script, checks if all the required functions
443  * are defined and runs the 'init' function. The init function will inform
444  * us what the scripts needs are.
445  *
446  * \param filename filename of lua script file
447  * \param options struct to pass script requirements/options back to caller
448  * \retval errcode 0 ok, -1 error
449  */
450 static int LuaScriptInit(const char *filename, LogLuaScriptOptions *options) {
451  int status;
452 
453  lua_State *luastate = LuaGetState();
454  if (luastate == NULL)
455  goto error;
456  luaL_openlibs(luastate);
457 
458  /* hackish, needed to allow unittests to pass buffers as scripts instead of files */
459 #if 0//def UNITTESTS
460  if (ut_script != NULL) {
461  status = luaL_loadbuffer(luastate, ut_script, strlen(ut_script), "unittest");
462  if (status) {
463  SCLogError(SC_ERR_LUA_ERROR, "couldn't load file: %s", lua_tostring(luastate, -1));
464  goto error;
465  }
466  } else {
467 #endif
468  status = luaL_loadfile(luastate, filename);
469  if (status) {
470  SCLogError(SC_ERR_LUA_ERROR, "couldn't load file: %s", lua_tostring(luastate, -1));
471  goto error;
472  }
473 #if 0//def UNITTESTS
474  }
475 #endif
476 
477  /* prime the script (or something) */
478  if (lua_pcall(luastate, 0, 0, 0) != 0) {
479  SCLogError(SC_ERR_LUA_ERROR, "couldn't prime file: %s", lua_tostring(luastate, -1));
480  goto error;
481  }
482 
483  lua_getglobal(luastate, "init");
484  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
485  SCLogError(SC_ERR_LUA_ERROR, "no init function in script");
486  goto error;
487  }
488 
489  lua_newtable(luastate); /* stack at -1 */
490  if (lua_gettop(luastate) == 0 || lua_type(luastate, 2) != LUA_TTABLE) {
491  SCLogError(SC_ERR_LUA_ERROR, "no table setup");
492  goto error;
493  }
494 
495  lua_pushliteral(luastate, "script_api_ver");
496  lua_pushnumber (luastate, 1);
497  lua_settable(luastate, -3);
498 
499  if (lua_pcall(luastate, 1, 1, 0) != 0) {
500  SCLogError(SC_ERR_LUA_ERROR, "couldn't run script 'init' function: %s", lua_tostring(luastate, -1));
501  goto error;
502  }
503 
504  /* process returns from script */
505  if (lua_gettop(luastate) == 0) {
506  SCLogError(SC_ERR_LUA_ERROR, "init function in script should return table, nothing returned");
507  goto error;
508  }
509  if (lua_type(luastate, 1) != LUA_TTABLE) {
510  SCLogError(SC_ERR_LUA_ERROR, "init function in script should return table, returned is not table");
511  goto error;
512  }
513 
514  lua_pushnil(luastate);
515  const char *k, *v;
516  while (lua_next(luastate, -2)) {
517  k = lua_tostring(luastate, -2);
518  if (k == NULL)
519  continue;
520 
521  v = lua_tostring(luastate, -1);
522  lua_pop(luastate, 1);
523  if (v == NULL)
524  continue;
525 
526  SCLogDebug("k='%s', v='%s'", k, v);
527 
528  if (strcmp(k,"protocol") == 0 && strcmp(v, "http") == 0)
529  options->alproto = ALPROTO_HTTP;
530  else if (strcmp(k,"protocol") == 0 && strcmp(v, "dns") == 0)
531  options->alproto = ALPROTO_DNS;
532  else if (strcmp(k,"protocol") == 0 && strcmp(v, "tls") == 0)
533  options->alproto = ALPROTO_TLS;
534  else if (strcmp(k,"protocol") == 0 && strcmp(v, "ssh") == 0)
535  options->alproto = ALPROTO_SSH;
536  else if (strcmp(k,"protocol") == 0 && strcmp(v, "smtp") == 0)
537  options->alproto = ALPROTO_SMTP;
538  else if (strcmp(k, "type") == 0 && strcmp(v, "packet") == 0)
539  options->packet = 1;
540  else if (strcmp(k, "filter") == 0 && strcmp(v, "alerts") == 0)
541  options->alerts = 1;
542  else if (strcmp(k, "type") == 0 && strcmp(v, "file") == 0)
543  options->file = 1;
544  else if (strcmp(k, "type") == 0 && strcmp(v, "streaming") == 0)
545  options->streaming = 1;
546  else if (strcmp(k, "type") == 0 && strcmp(v, "flow") == 0)
547  options->flow = 1;
548  else if (strcmp(k, "filter") == 0 && strcmp(v, "tcp") == 0)
549  options->tcp_data = 1;
550  else if (strcmp(k, "type") == 0 && strcmp(v, "stats") == 0)
551  options->stats = 1;
552  else
553  SCLogInfo("unknown key and/or value: k='%s', v='%s'", k, v);
554  }
555 
556  if (((options->alproto != ALPROTO_UNKNOWN)) + options->packet + options->file > 1) {
557  SCLogError(SC_ERR_LUA_ERROR, "invalid combination of 'needs' in the script");
558  goto error;
559  }
560 
561  lua_getglobal(luastate, "setup");
562  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
563  SCLogError(SC_ERR_LUA_ERROR, "no setup function in script");
564  goto error;
565  }
566 
567  lua_getglobal(luastate, "log");
568  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
569  SCLogError(SC_ERR_LUA_ERROR, "no log function in script");
570  goto error;
571  }
572 
573  lua_getglobal(luastate, "deinit");
574  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
575  SCLogError(SC_ERR_LUA_ERROR, "no deinit function in script");
576  goto error;
577  }
578 
579  LuaReturnState(luastate);
580  return 0;
581 error:
582  if (luastate)
583  LuaReturnState(luastate);
584  return -1;
585 }
586 
587 /** \brief setup a luastate for use at runtime
588  *
589  * This loads the script, primes it and then runs the 'setup' function.
590  *
591  * \retval state Returns the set up luastate on success, NULL on error
592  */
593 static lua_State *LuaScriptSetup(const char *filename)
594 {
595  lua_State *luastate = LuaGetState();
596  if (luastate == NULL) {
597  SCLogError(SC_ERR_LUA_ERROR, "luaL_newstate failed");
598  goto error;
599  }
600 
601  luaL_openlibs(luastate);
602 
603  int status;
604  /* hackish, needed to allow unittests to pass buffers as scripts instead of files */
605 #if 0//def UNITTESTS
606  if (ut_script != NULL) {
607  status = luaL_loadbuffer(t->luastate, ut_script, strlen(ut_script), "unittest");
608  if (status) {
609  SCLogError(SC_ERR_LUA_ERROR, "couldn't load file: %s", lua_tostring(t->luastate, -1));
610  goto error;
611  }
612  } else {
613 #endif
614  status = luaL_loadfile(luastate, filename);
615  if (status) {
616  SCLogError(SC_ERR_LUA_ERROR, "couldn't load file: %s", lua_tostring(luastate, -1));
617  goto error;
618  }
619 #if 0//def UNITTESTS
620  }
621 #endif
622 
623  /* prime the script */
624  if (lua_pcall(luastate, 0, 0, 0) != 0) {
625  SCLogError(SC_ERR_LUA_ERROR, "couldn't prime file: %s", lua_tostring(luastate, -1));
626  goto error;
627  }
628 
629  lua_getglobal(luastate, "setup");
630 
631  /* register functions common to all */
632  LuaRegisterFunctions(luastate);
633  /* unconditionally register http function. They will only work
634  * if the tx is registered in the state at runtime though. */
635  LuaRegisterHttpFunctions(luastate);
636  LuaRegisterDnsFunctions(luastate);
637  LuaRegisterJa3Functions(luastate);
638  LuaRegisterTlsFunctions(luastate);
639  LuaRegisterSshFunctions(luastate);
640  LuaRegisterSmtpFunctions(luastate);
641 
642  if (lua_pcall(luastate, 0, 0, 0) != 0) {
643  SCLogError(SC_ERR_LUA_ERROR, "couldn't run script 'setup' function: %s", lua_tostring(luastate, -1));
644  goto error;
645  }
646 
647  SCLogDebug("lua_State %p is set up", luastate);
648  return luastate;
649 error:
650  if (luastate)
651  LuaReturnState(luastate);
652  return NULL;
653 }
654 
655 static void LogLuaSubFree(OutputCtx *oc) {
656  if (oc->data)
657  SCFree(oc->data);
658  SCFree(oc);
659 }
660 
661 /** \brief initialize output for a script instance
662  *
663  * Runs script 'setup' function.
664  */
665 static OutputInitResult OutputLuaLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
666 {
667  OutputInitResult result = { NULL, false };
668  if (conf == NULL)
669  return result;
670 
671  LogLuaCtx *lua_ctx = SCMalloc(sizeof(LogLuaCtx));
672  if (unlikely(lua_ctx == NULL))
673  return result;
674  memset(lua_ctx, 0x00, sizeof(*lua_ctx));
675 
676  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
677  if (unlikely(output_ctx == NULL)) {
678  SCFree(lua_ctx);
679  return result;
680  }
681 
682  SCMutexInit(&lua_ctx->m, NULL);
683 
684  const char *dir = "";
685  if (parent_ctx && parent_ctx->data) {
686  LogLuaMasterCtx *mc = parent_ctx->data;
687  dir = mc->path;
688  }
689 
690  char path[PATH_MAX] = "";
691  int ret = snprintf(path, sizeof(path),"%s%s%s", dir, strlen(dir) ? "/" : "", conf->val);
692  if (ret < 0 || ret == sizeof(path)) {
693  SCLogError(SC_ERR_SPRINTF,"failed to construct lua script path");
694  goto error;
695  }
696  SCLogDebug("script full path %s", path);
697 
698  SCMutexLock(&lua_ctx->m);
699  lua_ctx->luastate = LuaScriptSetup(path);
700  SCMutexUnlock(&lua_ctx->m);
701  if (lua_ctx->luastate == NULL)
702  goto error;
703 
704  SCLogDebug("lua_ctx %p", lua_ctx);
705 
706  output_ctx->data = lua_ctx;
707  output_ctx->DeInit = LogLuaSubFree;
708 
709  result.ctx = output_ctx;
710  result.ok = true;
711  return result;
712 error:
713  SCMutexDestroy(&lua_ctx->m);
714  SCFree(lua_ctx);
715  SCFree(output_ctx);
716  return result;
717 }
718 
719 static void LogLuaMasterFree(OutputCtx *oc)
720 {
721  if (oc->data)
722  SCFree(oc->data);
723 
724  OutputModule *om, *tom;
725  TAILQ_FOREACH_SAFE(om, &oc->submodules, entries, tom) {
726  SCFree(om);
727  }
728  SCFree(oc);
729 }
730 
731 /** \internal
732  * \brief initialize output instance for lua module
733  *
734  * Parses nested script list, primes them to find out what they
735  * inspect, then fills the OutputCtx::submodules list with the
736  * proper Logger function for the data type the script needs.
737  */
738 static OutputInitResult OutputLuaLogInit(ConfNode *conf)
739 {
740  OutputInitResult result = { NULL, false };
741  const char *dir = ConfNodeLookupChildValue(conf, "scripts-dir");
742  if (dir == NULL)
743  dir = "";
744 
745  ConfNode *scripts = ConfNodeLookupChild(conf, "scripts");
746  if (scripts == NULL) {
747  /* No "outputs" section in the configuration. */
748  SCLogInfo("scripts not defined");
749  return result;
750  }
751 
752  /* global output ctx setup */
753  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
754  if (unlikely(output_ctx == NULL)) {
755  return result;
756  }
757  output_ctx->DeInit = LogLuaMasterFree;
758  output_ctx->data = SCCalloc(1, sizeof(LogLuaMasterCtx));
759  if (unlikely(output_ctx->data == NULL)) {
760  SCFree(output_ctx);
761  return result;
762  }
763  LogLuaMasterCtx *master_config = output_ctx->data;
764  strlcpy(master_config->path, dir, sizeof(master_config->path));
765  TAILQ_INIT(&output_ctx->submodules);
766 
767  /* check the enables scripts and set them up as submodules */
768  ConfNode *script;
769  TAILQ_FOREACH(script, &scripts->head, next) {
770  SCLogInfo("enabling script %s", script->val);
771  LogLuaScriptOptions opts;
772  memset(&opts, 0x00, sizeof(opts));
773 
774  char path[PATH_MAX] = "";
775  snprintf(path, sizeof(path),"%s%s%s", dir, strlen(dir) ? "/" : "", script->val);
776  SCLogDebug("script full path %s", path);
777 
778  int r = LuaScriptInit(path, &opts);
779  if (r != 0) {
780  SCLogError(SC_ERR_LUA_ERROR, "couldn't initialize scipt");
781  goto error;
782  }
783 
784  /* create an OutputModule for this script, based
785  * on it's needs. */
786  OutputModule *om = SCCalloc(1, sizeof(*om));
787  if (om == NULL) {
788  SCLogError(SC_ERR_MEM_ALLOC, "calloc() failed");
789  goto error;
790  }
791 
792  om->name = MODULE_NAME;
793  om->conf_name = script->val;
794  om->InitSubFunc = OutputLuaLogInitSub;
795  om->ThreadInit = LuaLogThreadInit;
796  om->ThreadDeinit = LuaLogThreadDeinit;
797 
798  if (opts.alproto == ALPROTO_HTTP && opts.streaming) {
799  om->StreamingLogFunc = LuaStreamingLogger;
801  om->alproto = ALPROTO_HTTP;
804  } else if (opts.alproto == ALPROTO_HTTP) {
805  om->TxLogFunc = LuaTxLogger;
806  om->alproto = ALPROTO_HTTP;
807  om->ts_log_progress = -1;
808  om->tc_log_progress = -1;
810  } else if (opts.alproto == ALPROTO_TLS) {
811  om->TxLogFunc = LuaTxLogger;
812  om->alproto = ALPROTO_TLS;
815  } else if (opts.alproto == ALPROTO_DNS) {
816  om->TxLogFunc = LuaTxLogger;
817  om->alproto = ALPROTO_DNS;
818  om->ts_log_progress = -1;
819  om->tc_log_progress = -1;
822  } else if (opts.alproto == ALPROTO_SSH) {
823  om->TxLogFunc = LuaTxLogger;
824  om->alproto = ALPROTO_SSH;
827  } else if (opts.alproto == ALPROTO_SMTP) {
828  om->TxLogFunc = LuaTxLogger;
829  om->alproto = ALPROTO_SMTP;
830  om->ts_log_progress = -1;
831  om->tc_log_progress = -1;
833  } else if (opts.packet && opts.alerts) {
834  om->PacketLogFunc = LuaPacketLoggerAlerts;
835  om->PacketConditionFunc = LuaPacketConditionAlerts;
836  } else if (opts.packet && opts.alerts == 0) {
837  om->PacketLogFunc = LuaPacketLogger;
838  om->PacketConditionFunc = LuaPacketCondition;
839  } else if (opts.file) {
840  om->FileLogFunc = LuaFileLogger;
842  } else if (opts.streaming && opts.tcp_data) {
843  om->StreamingLogFunc = LuaStreamingLogger;
845  } else if (opts.flow) {
846  om->FlowLogFunc = LuaFlowLogger;
847  } else if (opts.stats) {
848  om->StatsLogFunc = LuaStatsLogger;
849  } else {
850  SCLogError(SC_ERR_LUA_ERROR, "failed to setup thread module");
851  SCFree(om);
852  goto error;
853  }
854 
855  TAILQ_INSERT_TAIL(&output_ctx->submodules, om, entries);
856  }
857 
858  result.ctx = output_ctx;
859  result.ok = true;
860  return result;
861 
862 error:
863  if (output_ctx->DeInit)
864  output_ctx->DeInit(output_ctx);
865 
866  int failure_fatal = 0;
867  if (ConfGetBool("engine.init-failure-fatal", &failure_fatal) != 1) {
868  SCLogDebug("ConfGetBool could not load the value.");
869  }
870  if (failure_fatal) {
872  "Error during setup of lua output. Details should be "
873  "described in previous error messages. Shutting down...");
874  exit(EXIT_FAILURE);
875  }
876 
877  return result;
878 }
879 
880 /** \internal
881  * \brief Run the scripts 'deinit' function
882  */
883 static void OutputLuaLogDoDeinit(LogLuaCtx *lua_ctx)
884 {
885  lua_State *luastate = lua_ctx->luastate;
886 
887  lua_getglobal(luastate, "deinit");
888  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
889  SCLogError(SC_ERR_LUA_ERROR, "no deinit function in script");
890  return;
891  }
892  //LuaPrintStack(luastate);
893 
894  if (lua_pcall(luastate, 0, 0, 0) != 0) {
895  SCLogError(SC_ERR_LUA_ERROR, "couldn't run script 'deinit' function: %s", lua_tostring(luastate, -1));
896  return;
897  }
898  LuaReturnState(luastate);
899 }
900 
901 /** \internal
902  * \brief Initialize the thread storage for lua
903  *
904  * Currently only stores a pointer to the global LogLuaCtx
905  */
906 static TmEcode LuaLogThreadInit(ThreadVars *t, const void *initdata, void **data)
907 {
908  LogLuaThreadCtx *td = SCMalloc(sizeof(*td));
909  if (unlikely(td == NULL))
910  return TM_ECODE_FAILED;
911  memset(td, 0, sizeof(*td));
912 
913  if (initdata == NULL) {
914  SCLogDebug("Error getting context for LuaLog. \"initdata\" argument NULL");
915  SCFree(td);
916  return TM_ECODE_FAILED;
917  }
918 
919  LogLuaCtx *lua_ctx = ((OutputCtx *)initdata)->data;
920  SCLogDebug("lua_ctx %p", lua_ctx);
921  td->lua_ctx = lua_ctx;
922  *data = (void *)td;
923  return TM_ECODE_OK;
924 }
925 
926 /** \internal
927  * \brief Deinit the thread storage for lua
928  *
929  * Calls OutputLuaLogDoDeinit if no-one else already did.
930  */
931 static TmEcode LuaLogThreadDeinit(ThreadVars *t, void *data)
932 {
933  LogLuaThreadCtx *td = (LogLuaThreadCtx *)data;
934  if (td == NULL) {
935  return TM_ECODE_OK;
936  }
937 
938  SCMutexLock(&td->lua_ctx->m);
939  if (td->lua_ctx->deinit_once == 0) {
940  OutputLuaLogDoDeinit(td->lua_ctx);
941  td->lua_ctx->deinit_once = 1;
942  }
943  SCMutexUnlock(&td->lua_ctx->m);
944 
945  /* clear memory */
946  memset(td, 0, sizeof(*td));
947 
948  SCFree(td);
949  return TM_ECODE_OK;
950 }
951 
952 void LuaLogRegister(void) {
953  /* register as separate module */
954  OutputRegisterModule(MODULE_NAME, "lua", OutputLuaLogInit);
955 }
956 
957 #else
958 
959 void LuaLogRegister (void) {
960  /* no-op */
961 }
962 
963 #endif
StreamingLogger StreamingLogFunc
Definition: output.h:69
#define SCMutex
char * known_proto[256]
uint16_t flags
int ts_log_progress
Definition: output.h:74
uint8_t SCProtoNameValid(uint16_t proto)
Function to check if the received protocol number is valid and do we have corresponding name entry fo...
#define SCLogDebug(...)
Definition: util-debug.h:335
struct Flow_ * flow
Definition: decode.h:443
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:350
struct HtpBodyChunk_ * next
#define PACKET_ALERT_FLAG_TX
Definition: decode.h:284
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
#define BUG_ON(x)
uint8_t proto
Definition: flow.h:343
#define FALSE
#define unlikely(expr)
Definition: util-optimize.h:35
ThreadDeinitFunc ThreadDeinit
Definition: output.h:59
PacketLogger PacketLogFunc
Definition: output.h:62
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:517
uint64_t pvalue
Definition: output-stats.h:33
#define TAILQ_FOREACH_SAFE(var, head, field, tvar)
Definition: queue.h:356
int tc_log_progress
Definition: output.h:73
#define PKT_IS_IPV6(p)
Definition: decode.h:250
OutputInitSubFunc InitSubFunc
Definition: output.h:56
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
uint16_t AppProto
char * val
Definition: conf.h:34
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:815
#define TRUE
#define SCMutexLock(mut)
#define PKT_IS_IPV4(p)
Definition: decode.h:249
ThreadInitFunc ThreadInit
Definition: output.h:58
PacketAlert alerts[PACKET_ALERT_MAX]
Definition: decode.h:292
const struct Signature_ * s
Definition: decode.h:271
void AppLayerHtpNeedFileInspection(void)
Sets a flag that informs the HTP app layer that some module in the engine needs the http request file...
void * alstate
Definition: flow.h:433
#define FILE_LOGGED
Definition: util-file.h:44
#define SCCalloc(nm, a)
Definition: util-mem.h:197
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:843
#define SCMutexUnlock(mut)
uint16_t flags
Definition: util-file.h:65
uint8_t proto
Definition: decode.h:428
#define TAILQ_INIT(head)
Definition: queue.h:370
TxLogger TxLogFunc
Definition: output.h:64
void AppLayerHtpEnableRequestBodyCallback(void)
Sets a flag that informs the HTP app layer that some module in the engine needs the http request body...
AppProto alproto
Definition: output.h:71
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:294
void LuaLogRegister(void)
Definition: output-lua.c:959
void AppLayerParserRegisterLogger(uint8_t ipproto, AppProto alproto)
const char * name
Definition: output.h:52
enum OutputStreamingType stream_type
Definition: output.h:72
uint64_t txid
Definition: util-file.h:69
#define SCEnter(...)
Definition: util-debug.h:337
#define MODULE_NAME
void AppLayerHtpEnableResponseBodyCallback(void)
Sets a flag that informs the HTP app layer that some module in the engine needs the http request body...
const char * tm_name
Definition: output-stats.h:31
#define SCMutexInit(mut, mutattrs)
FlowLogger FlowLogFunc
Definition: output.h:68
uint8_t proto
#define SCReturnInt(x)
Definition: util-debug.h:341
uint64_t tx_id
Definition: decode.h:272
#define TAILQ_INSERT_TAIL(head, elm, field)
Definition: queue.h:385
Definition: conf.h:32
OutputCtx * ctx
Definition: output.h:42
#define SCMalloc(a)
Definition: util-mem.h:166
StatsRecord * stats
Definition: output-stats.h:37
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:254
#define SCFree(a)
Definition: util-mem.h:228
uint64_t value
Definition: output-stats.h:32
uint16_t tx_id
uint8_t flags
Definition: decode.h:270
void * data
Definition: tm-modules.h:81
#define IP_GET_IPPROTO(p)
Definition: decode.h:261
FileLogger FileLogFunc
Definition: output.h:66
SCMutex m
Definition: flow-hash.h:105
PacketAlerts alerts
Definition: decode.h:555
const char * conf_name
Definition: output.h:53
StatsLogger StatsLogFunc
Definition: output.h:70
#define OUTPUT_STREAMING_FLAG_TRANSACTION
uint16_t cnt
Definition: decode.h:291
PacketLogCondition PacketConditionFunc
Definition: output.h:63
Per thread variable structure.
Definition: threadvars.h:57
const char * name
Definition: output-stats.h:30
struct timeval ts
Definition: decode.h:449
AppProto alproto
application level protocol
Definition: flow.h:404
uint32_t nstats
Definition: output-stats.h:39
Flow data structure.
Definition: flow.h:324
void CreateTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:237
void OutputRegisterModule(const char *, const char *, OutputInitFunc)
#define SCMutexDestroy