suricata
output-lua.c
Go to the documentation of this file.
1 /* Copyright (C) 2014-2020 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  */
24 
25 #include "suricata-common.h"
26 #include "debug.h"
27 #include "pkt-var.h"
28 #include "conf.h"
29 
30 #include "threads.h"
31 #include "threadvars.h"
32 #include "tm-threads.h"
33 
34 #include "util-print.h"
35 #include "util-unittest.h"
36 
37 #include "util-debug.h"
38 
39 #include "output.h"
40 #include "app-layer-htp.h"
41 #include "app-layer.h"
42 #include "app-layer-ssl.h"
43 #include "app-layer-ssh.h"
44 #include "app-layer-parser.h"
45 #include "util-privs.h"
46 #include "util-buffer.h"
47 #include "util-proto-name.h"
48 #include "util-logopenfile.h"
49 #include "util-time.h"
50 
51 #include "output-lua.h"
52 
53 #ifdef HAVE_LUA
54 
55 #include <lua.h>
56 #include <lualib.h>
57 #include <lauxlib.h>
58 
59 #include "util-lua.h"
60 #include "util-lua-common.h"
61 #include "util-lua-http.h"
62 #include "util-lua-dns.h"
63 #include "util-lua-ja3.h"
64 #include "util-lua-tls.h"
65 #include "util-lua-ssh.h"
66 #include "util-lua-hassh.h"
67 #include "util-lua-smtp.h"
68 
69 #define MODULE_NAME "LuaLog"
70 
71 /** \brief structure containing global config
72  * The OutputLuaLogInitSub which is run per script
73  * can access this to get global config info through
74  * it's parent_ctx->data ptr.
75  */
76 typedef struct LogLuaMasterCtx_ {
77  char path[PATH_MAX]; /**< contains script-dir */
78 } LogLuaMasterCtx;
79 
80 typedef struct LogLuaCtx_ {
81  SCMutex m;
82  lua_State *luastate;
83  int deinit_once;
84 } LogLuaCtx;
85 
86 typedef struct LogLuaThreadCtx_ {
87  LogLuaCtx *lua_ctx;
88 } LogLuaThreadCtx;
89 
90 static TmEcode LuaLogThreadInit(ThreadVars *t, const void *initdata, void **data);
91 static TmEcode LuaLogThreadDeinit(ThreadVars *t, void *data);
92 
93 /** \internal
94  * \brief TX logger for lua scripts
95  *
96  * A single call to this function will run one script on a single
97  * transaction.
98  *
99  * NOTE: The flow (f) also referenced by p->flow is locked.
100  */
101 static int LuaTxLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *alstate, void *txptr, uint64_t tx_id)
102 {
103  SCEnter();
104 
105  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
106 
107  SCMutexLock(&td->lua_ctx->m);
108 
109  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
110  LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
111  LuaStateSetTX(td->lua_ctx->luastate, txptr);
112  LuaStateSetFlow(td->lua_ctx->luastate, f);
113 
114  /* prepare data to pass to script */
115  lua_getglobal(td->lua_ctx->luastate, "log");
116  lua_newtable(td->lua_ctx->luastate);
117  LuaPushTableKeyValueInt(td->lua_ctx->luastate, "tx_id", (int)(tx_id));
118 
119  int retval = lua_pcall(td->lua_ctx->luastate, 1, 0, 0);
120  if (retval != 0) {
121  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
122  }
123 
124  SCMutexUnlock(&td->lua_ctx->m);
125  SCReturnInt(0);
126 }
127 
128 /** \internal
129  * \brief Streaming logger for lua scripts
130  *
131  * Hooks into the Streaming Logger API. Gets called for each chunk of new
132  * streaming data.
133  */
134 static int LuaStreamingLogger(ThreadVars *tv, void *thread_data, const Flow *f,
135  const uint8_t *data, uint32_t data_len, uint64_t tx_id, uint8_t flags)
136 {
137  SCEnter();
138 
139  void *txptr = NULL;
140  LuaStreamingBuffer b = { data, data_len, flags };
141 
142  SCLogDebug("flags %02x", flags);
143 
145  if (f && f->alstate)
146  txptr = AppLayerParserGetTx(f->proto, f->alproto, f->alstate, tx_id);
147  }
148 
149  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
150 
151  SCMutexLock(&td->lua_ctx->m);
152 
153  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
155  LuaStateSetTX(td->lua_ctx->luastate, txptr);
156  LuaStateSetFlow(td->lua_ctx->luastate, (Flow *)f);
157  LuaStateSetStreamingBuffer(td->lua_ctx->luastate, &b);
158 
159  /* prepare data to pass to script */
160  lua_getglobal(td->lua_ctx->luastate, "log");
161  lua_newtable(td->lua_ctx->luastate);
162 
164  LuaPushTableKeyValueInt(td->lua_ctx->luastate, "tx_id", (int)(tx_id));
165 
166  int retval = lua_pcall(td->lua_ctx->luastate, 1, 0, 0);
167  if (retval != 0) {
168  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
169  }
170 
171  SCMutexUnlock(&td->lua_ctx->m);
172 
174 }
175 
176 /** \internal
177  * \brief Packet Logger for lua scripts, for alerts
178  *
179  * A single call to this function will run one script for a single
180  * packet. If it is called, it means that the registered condition
181  * function has returned TRUE.
182  *
183  * The script is called once for each alert stored in the packet.
184  *
185  * NOTE: p->flow is UNlocked
186  */
187 static int LuaPacketLoggerAlerts(ThreadVars *tv, void *thread_data, const Packet *p)
188 {
189  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
190 
191  char timebuf[64];
192  CreateTimeString(&p->ts, timebuf, sizeof(timebuf));
193 
194  if (!(PKT_IS_IPV4(p)) && !(PKT_IS_IPV6(p))) {
195  /* decoder event */
196  goto not_supported;
197  }
198 
199  /* loop through alerts stored in the packet */
200  SCMutexLock(&td->lua_ctx->m);
201  uint16_t cnt;
202  for (cnt = 0; cnt < p->alerts.cnt; cnt++) {
203  const PacketAlert *pa = &p->alerts.alerts[cnt];
204  if (unlikely(pa->s == NULL)) {
205  continue;
206  }
207 
208  lua_getglobal(td->lua_ctx->luastate, "log");
209 
210  void *txptr = NULL;
211  if (p->flow && p->flow->alstate && (pa->flags & PACKET_ALERT_FLAG_TX))
212  txptr = AppLayerParserGetTx(p->proto, p->flow->alproto, p->flow->alstate, pa->tx_id);
213 
214  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
215  LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
216  LuaStateSetTX(td->lua_ctx->luastate, txptr);
217  LuaStateSetFlow(td->lua_ctx->luastate, p->flow);
218  LuaStateSetPacketAlert(td->lua_ctx->luastate, (PacketAlert *)pa);
219 
220  /* prepare data to pass to script */
221  //lua_newtable(td->lua_ctx->luastate);
222 
223  int retval = lua_pcall(td->lua_ctx->luastate, 0, 0, 0);
224  if (retval != 0) {
225  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
226  }
227  }
228  SCMutexUnlock(&td->lua_ctx->m);
229 not_supported:
230  SCReturnInt(0);
231 }
232 
233 static int LuaPacketConditionAlerts(ThreadVars *tv, const Packet *p)
234 {
235  if (p->alerts.cnt > 0)
236  return TRUE;
237  return FALSE;
238 }
239 
240 /** \internal
241  * \brief Packet Logger for lua scripts, for packets
242  *
243  * A single call to this function will run one script for a single
244  * packet. If it is called, it means that the registered condition
245  * function has returned TRUE.
246  *
247  * The script is called once for each packet.
248  *
249  * NOTE: p->flow is UNlocked
250  */
251 static int LuaPacketLogger(ThreadVars *tv, void *thread_data, const Packet *p)
252 {
253  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
254 
255  char timebuf[64];
256 
257  if ((!(PKT_IS_IPV4(p))) && (!(PKT_IS_IPV6(p)))) {
258  goto not_supported;
259  }
260 
261  CreateTimeString(&p->ts, timebuf, sizeof(timebuf));
262 
263  /* loop through alerts stored in the packet */
264  SCMutexLock(&td->lua_ctx->m);
265  lua_getglobal(td->lua_ctx->luastate, "log");
266 
267  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
268  LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
269  LuaStateSetFlow(td->lua_ctx->luastate, p->flow);
270 
271  /* prepare data to pass to script */
272  lua_newtable(td->lua_ctx->luastate);
273 
274  int retval = lua_pcall(td->lua_ctx->luastate, 1, 0, 0);
275  if (retval != 0) {
276  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
277  }
278  SCMutexUnlock(&td->lua_ctx->m);
279 not_supported:
280  SCReturnInt(0);
281 }
282 
283 static int LuaPacketCondition(ThreadVars *tv, const Packet *p)
284 {
285  return TRUE;
286 }
287 
288 /** \internal
289  * \brief File API Logger function for Lua scripts
290  *
291  * Executes a script once for one file.
292  *
293  * NOTE p->flow is locked at this point
294  */
295 static int LuaFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff, uint8_t dir)
296 {
297  SCEnter();
298  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
299 
300  if ((!(PKT_IS_IPV4(p))) && (!(PKT_IS_IPV6(p))))
301  return 0;
302 
303  BUG_ON(ff->flags & FILE_LOGGED);
304 
305  SCLogDebug("ff %p", ff);
306 
307  SCMutexLock(&td->lua_ctx->m);
308 
309  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
310  LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
311  if (p->flow && p->flow->alstate) {
312  void *txptr = AppLayerParserGetTx(p->proto, p->flow->alproto, p->flow->alstate, ff->txid);
313  if (txptr) {
314  LuaStateSetTX(td->lua_ctx->luastate, txptr);
315  }
316  }
317  LuaStateSetFlow(td->lua_ctx->luastate, p->flow);
318  LuaStateSetFile(td->lua_ctx->luastate, (File *)ff);
319 
320  /* get the lua function to call */
321  lua_getglobal(td->lua_ctx->luastate, "log");
322 
323  int retval = lua_pcall(td->lua_ctx->luastate, 0, 0, 0);
324  if (retval != 0) {
325  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
326  }
327  SCMutexUnlock(&td->lua_ctx->m);
328  return 0;
329 }
330 
331 /** \internal
332  * \brief Flow API Logger function for Lua scripts
333  *
334  * Executes a script once for one flow
335  *
336  * Note: flow 'f' is locked
337  */
338 static int LuaFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
339 {
340  SCEnter();
341  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
342 
343  SCLogDebug("f %p", f);
344 
345  SCMutexLock(&td->lua_ctx->m);
346 
347  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
348  LuaStateSetFlow(td->lua_ctx->luastate, f);
349 
350  /* get the lua function to call */
351  lua_getglobal(td->lua_ctx->luastate, "log");
352 
353  int retval = lua_pcall(td->lua_ctx->luastate, 0, 0, 0);
354  if (retval != 0) {
355  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
356  }
357  SCMutexUnlock(&td->lua_ctx->m);
358  return 0;
359 }
360 
361 
362 
363 static int LuaStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable *st)
364 {
365  SCEnter();
366  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
367 
368  SCMutexLock(&td->lua_ctx->m);
369 
370  lua_State *luastate = td->lua_ctx->luastate;
371  /* get the lua function to call */
372  lua_getglobal(td->lua_ctx->luastate, "log");
373 
374  /* create lua array, which is really just a table. The key is an int (1-x),
375  * the value another table with named fields: name, tm_name, value, pvalue.
376  * { 1, { name=<name>, tmname=<tm_name>, value=<value>, pvalue=<pvalue>}}
377  * { 2, { name=<name>, tmname=<tm_name>, value=<value>, pvalue=<pvalue>}}
378  * etc
379  */
380  lua_newtable(luastate);
381  uint32_t u = 0;
382  for (; u < st->nstats; u++) {
383  lua_pushinteger(luastate, u + 1);
384 
385  lua_newtable(luastate);
386 
387  lua_pushstring(luastate, "name");
388  lua_pushstring(luastate, st->stats[u].name);
389  lua_settable(luastate, -3);
390 
391  lua_pushstring(luastate, "tmname");
392  lua_pushstring(luastate, st->stats[u].tm_name);
393  lua_settable(luastate, -3);
394 
395  lua_pushstring(luastate, "value");
396  lua_pushinteger(luastate, st->stats[u].value);
397  lua_settable(luastate, -3);
398 
399  lua_pushstring(luastate, "pvalue");
400  lua_pushinteger(luastate, st->stats[u].pvalue);
401  lua_settable(luastate, -3);
402 
403  lua_settable(luastate, -3);
404  }
405 
406  int retval = lua_pcall(td->lua_ctx->luastate, 1, 0, 0);
407  if (retval != 0) {
408  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
409  }
410  SCMutexUnlock(&td->lua_ctx->m);
411  return 0;
412 
413 }
414 
415 typedef struct LogLuaScriptOptions_ {
416  AppProto alproto;
417  int packet;
418  int alerts;
419  int file;
420  int streaming;
421  int tcp_data;
422  int http_body;
423  int flow;
424  int stats;
425 } LogLuaScriptOptions;
426 
427 /** \brief load and evaluate the script
428  *
429  * This function parses the script, checks if all the required functions
430  * are defined and runs the 'init' function. The init function will inform
431  * us what the scripts needs are.
432  *
433  * \param filename filename of lua script file
434  * \param options struct to pass script requirements/options back to caller
435  * \retval errcode 0 ok, -1 error
436  */
437 static int LuaScriptInit(const char *filename, LogLuaScriptOptions *options) {
438  int status;
439 
440  lua_State *luastate = LuaGetState();
441  if (luastate == NULL)
442  goto error;
443  luaL_openlibs(luastate);
444 
445  /* hackish, needed to allow unittests to pass buffers as scripts instead of files */
446 #if 0//def UNITTESTS
447  if (ut_script != NULL) {
448  status = luaL_loadbuffer(luastate, ut_script, strlen(ut_script), "unittest");
449  if (status) {
450  SCLogError(SC_ERR_LUA_ERROR, "couldn't load file: %s", lua_tostring(luastate, -1));
451  goto error;
452  }
453  } else {
454 #endif
455  status = luaL_loadfile(luastate, filename);
456  if (status) {
457  SCLogError(SC_ERR_LUA_ERROR, "couldn't load file: %s", lua_tostring(luastate, -1));
458  goto error;
459  }
460 #if 0//def UNITTESTS
461  }
462 #endif
463 
464  /* prime the script (or something) */
465  if (lua_pcall(luastate, 0, 0, 0) != 0) {
466  SCLogError(SC_ERR_LUA_ERROR, "couldn't prime file: %s", lua_tostring(luastate, -1));
467  goto error;
468  }
469 
470  lua_getglobal(luastate, "init");
471  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
472  SCLogError(SC_ERR_LUA_ERROR, "no init function in script");
473  goto error;
474  }
475 
476  lua_newtable(luastate); /* stack at -1 */
477  if (lua_gettop(luastate) == 0 || lua_type(luastate, 2) != LUA_TTABLE) {
478  SCLogError(SC_ERR_LUA_ERROR, "no table setup");
479  goto error;
480  }
481 
482  lua_pushliteral(luastate, "script_api_ver");
483  lua_pushnumber (luastate, 1);
484  lua_settable(luastate, -3);
485 
486  if (lua_pcall(luastate, 1, 1, 0) != 0) {
487  SCLogError(SC_ERR_LUA_ERROR, "couldn't run script 'init' function: %s", lua_tostring(luastate, -1));
488  goto error;
489  }
490 
491  /* process returns from script */
492  if (lua_gettop(luastate) == 0) {
493  SCLogError(SC_ERR_LUA_ERROR, "init function in script should return table, nothing returned");
494  goto error;
495  }
496  if (lua_type(luastate, 1) != LUA_TTABLE) {
497  SCLogError(SC_ERR_LUA_ERROR, "init function in script should return table, returned is not table");
498  goto error;
499  }
500 
501  lua_pushnil(luastate);
502  const char *k, *v;
503  while (lua_next(luastate, -2)) {
504  k = lua_tostring(luastate, -2);
505  if (k == NULL)
506  continue;
507 
508  v = lua_tostring(luastate, -1);
509  lua_pop(luastate, 1);
510  if (v == NULL)
511  continue;
512 
513  SCLogDebug("k='%s', v='%s'", k, v);
514 
515  if (strcmp(k,"protocol") == 0 && strcmp(v, "http") == 0)
516  options->alproto = ALPROTO_HTTP;
517  else if (strcmp(k,"protocol") == 0 && strcmp(v, "dns") == 0)
518  options->alproto = ALPROTO_DNS;
519  else if (strcmp(k,"protocol") == 0 && strcmp(v, "tls") == 0)
520  options->alproto = ALPROTO_TLS;
521  else if (strcmp(k,"protocol") == 0 && strcmp(v, "ssh") == 0)
522  options->alproto = ALPROTO_SSH;
523  else if (strcmp(k,"protocol") == 0 && strcmp(v, "smtp") == 0)
524  options->alproto = ALPROTO_SMTP;
525  else if (strcmp(k, "type") == 0 && strcmp(v, "packet") == 0)
526  options->packet = 1;
527  else if (strcmp(k, "filter") == 0 && strcmp(v, "alerts") == 0)
528  options->alerts = 1;
529  else if (strcmp(k, "type") == 0 && strcmp(v, "file") == 0)
530  options->file = 1;
531  else if (strcmp(k, "type") == 0 && strcmp(v, "streaming") == 0)
532  options->streaming = 1;
533  else if (strcmp(k, "type") == 0 && strcmp(v, "flow") == 0)
534  options->flow = 1;
535  else if (strcmp(k, "filter") == 0 && strcmp(v, "tcp") == 0)
536  options->tcp_data = 1;
537  else if (strcmp(k, "type") == 0 && strcmp(v, "stats") == 0)
538  options->stats = 1;
539  else
540  SCLogInfo("unknown key and/or value: k='%s', v='%s'", k, v);
541  }
542 
543  if (((options->alproto != ALPROTO_UNKNOWN)) + options->packet + options->file > 1) {
544  SCLogError(SC_ERR_LUA_ERROR, "invalid combination of 'needs' in the script");
545  goto error;
546  }
547 
548  lua_getglobal(luastate, "setup");
549  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
550  SCLogError(SC_ERR_LUA_ERROR, "no setup function in script");
551  goto error;
552  }
553 
554  lua_getglobal(luastate, "log");
555  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
556  SCLogError(SC_ERR_LUA_ERROR, "no log function in script");
557  goto error;
558  }
559 
560  lua_getglobal(luastate, "deinit");
561  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
562  SCLogError(SC_ERR_LUA_ERROR, "no deinit function in script");
563  goto error;
564  }
565 
566  LuaReturnState(luastate);
567  return 0;
568 error:
569  if (luastate)
570  LuaReturnState(luastate);
571  return -1;
572 }
573 
574 /** \brief setup a luastate for use at runtime
575  *
576  * This loads the script, primes it and then runs the 'setup' function.
577  *
578  * \retval state Returns the set up luastate on success, NULL on error
579  */
580 static lua_State *LuaScriptSetup(const char *filename)
581 {
582  lua_State *luastate = LuaGetState();
583  if (luastate == NULL) {
584  SCLogError(SC_ERR_LUA_ERROR, "luaL_newstate failed");
585  goto error;
586  }
587 
588  luaL_openlibs(luastate);
589 
590  int status;
591  /* hackish, needed to allow unittests to pass buffers as scripts instead of files */
592 #if 0//def UNITTESTS
593  if (ut_script != NULL) {
594  status = luaL_loadbuffer(t->luastate, ut_script, strlen(ut_script), "unittest");
595  if (status) {
596  SCLogError(SC_ERR_LUA_ERROR, "couldn't load file: %s", lua_tostring(t->luastate, -1));
597  goto error;
598  }
599  } else {
600 #endif
601  status = luaL_loadfile(luastate, filename);
602  if (status) {
603  SCLogError(SC_ERR_LUA_ERROR, "couldn't load file: %s", lua_tostring(luastate, -1));
604  goto error;
605  }
606 #if 0//def UNITTESTS
607  }
608 #endif
609 
610  /* prime the script */
611  if (lua_pcall(luastate, 0, 0, 0) != 0) {
612  SCLogError(SC_ERR_LUA_ERROR, "couldn't prime file: %s", lua_tostring(luastate, -1));
613  goto error;
614  }
615 
616  lua_getglobal(luastate, "setup");
617 
618  /* register functions common to all */
619  LuaRegisterFunctions(luastate);
620  /* unconditionally register http function. They will only work
621  * if the tx is registered in the state at runtime though. */
622  LuaRegisterHttpFunctions(luastate);
623  LuaRegisterDnsFunctions(luastate);
624  LuaRegisterJa3Functions(luastate);
625  LuaRegisterTlsFunctions(luastate);
626  LuaRegisterSshFunctions(luastate);
627  LuaRegisterHasshFunctions(luastate);
628  LuaRegisterSmtpFunctions(luastate);
629 
630  if (lua_pcall(luastate, 0, 0, 0) != 0) {
631  SCLogError(SC_ERR_LUA_ERROR, "couldn't run script 'setup' function: %s", lua_tostring(luastate, -1));
632  goto error;
633  }
634 
635  SCLogDebug("lua_State %p is set up", luastate);
636  return luastate;
637 error:
638  if (luastate)
639  LuaReturnState(luastate);
640  return NULL;
641 }
642 
643 static void LogLuaSubFree(OutputCtx *oc) {
644  if (oc->data)
645  SCFree(oc->data);
646  SCFree(oc);
647 }
648 
649 /** \brief initialize output for a script instance
650  *
651  * Runs script 'setup' function.
652  */
653 static OutputInitResult OutputLuaLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
654 {
655  OutputInitResult result = { NULL, false };
656  if (conf == NULL)
657  return result;
658 
659  LogLuaCtx *lua_ctx = SCMalloc(sizeof(LogLuaCtx));
660  if (unlikely(lua_ctx == NULL))
661  return result;
662  memset(lua_ctx, 0x00, sizeof(*lua_ctx));
663 
664  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
665  if (unlikely(output_ctx == NULL)) {
666  SCFree(lua_ctx);
667  return result;
668  }
669 
670  SCMutexInit(&lua_ctx->m, NULL);
671 
672  const char *dir = "";
673  if (parent_ctx && parent_ctx->data) {
674  LogLuaMasterCtx *mc = parent_ctx->data;
675  dir = mc->path;
676  }
677 
678  char path[PATH_MAX] = "";
679  int ret = snprintf(path, sizeof(path),"%s%s%s", dir, strlen(dir) ? "/" : "", conf->val);
680  if (ret < 0 || ret == sizeof(path)) {
681  SCLogError(SC_ERR_SPRINTF,"failed to construct lua script path");
682  goto error;
683  }
684  SCLogDebug("script full path %s", path);
685 
686  SCMutexLock(&lua_ctx->m);
687  lua_ctx->luastate = LuaScriptSetup(path);
688  SCMutexUnlock(&lua_ctx->m);
689  if (lua_ctx->luastate == NULL)
690  goto error;
691 
692  SCLogDebug("lua_ctx %p", lua_ctx);
693 
694  output_ctx->data = lua_ctx;
695  output_ctx->DeInit = LogLuaSubFree;
696 
697  result.ctx = output_ctx;
698  result.ok = true;
699  return result;
700 error:
701  SCMutexDestroy(&lua_ctx->m);
702  SCFree(lua_ctx);
703  SCFree(output_ctx);
704  return result;
705 }
706 
707 static void LogLuaMasterFree(OutputCtx *oc)
708 {
709  if (oc->data)
710  SCFree(oc->data);
711 
712  OutputModule *om, *tom;
713  TAILQ_FOREACH_SAFE(om, &oc->submodules, entries, tom) {
714  SCFree(om);
715  }
716  SCFree(oc);
717 }
718 
719 /** \internal
720  * \brief initialize output instance for lua module
721  *
722  * Parses nested script list, primes them to find out what they
723  * inspect, then fills the OutputCtx::submodules list with the
724  * proper Logger function for the data type the script needs.
725  */
726 static OutputInitResult OutputLuaLogInit(ConfNode *conf)
727 {
728  OutputInitResult result = { NULL, false };
729  const char *dir = ConfNodeLookupChildValue(conf, "scripts-dir");
730  if (dir == NULL)
731  dir = "";
732 
733  ConfNode *scripts = ConfNodeLookupChild(conf, "scripts");
734  if (scripts == NULL) {
735  /* No "outputs" section in the configuration. */
736  SCLogInfo("scripts not defined");
737  return result;
738  }
739 
740  /* global output ctx setup */
741  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
742  if (unlikely(output_ctx == NULL)) {
743  return result;
744  }
745  output_ctx->DeInit = LogLuaMasterFree;
746  output_ctx->data = SCCalloc(1, sizeof(LogLuaMasterCtx));
747  if (unlikely(output_ctx->data == NULL)) {
748  SCFree(output_ctx);
749  return result;
750  }
751  LogLuaMasterCtx *master_config = output_ctx->data;
752  strlcpy(master_config->path, dir, sizeof(master_config->path));
753  TAILQ_INIT(&output_ctx->submodules);
754 
755  /* check the enables scripts and set them up as submodules */
756  ConfNode *script;
757  TAILQ_FOREACH(script, &scripts->head, next) {
758  SCLogInfo("enabling script %s", script->val);
759  LogLuaScriptOptions opts;
760  memset(&opts, 0x00, sizeof(opts));
761 
762  char path[PATH_MAX] = "";
763  snprintf(path, sizeof(path),"%s%s%s", dir, strlen(dir) ? "/" : "", script->val);
764  SCLogDebug("script full path %s", path);
765 
766  int r = LuaScriptInit(path, &opts);
767  if (r != 0) {
768  SCLogError(SC_ERR_LUA_ERROR, "couldn't initialize scipt");
769  goto error;
770  }
771 
772  /* create an OutputModule for this script, based
773  * on it's needs. */
774  OutputModule *om = SCCalloc(1, sizeof(*om));
775  if (om == NULL) {
776  SCLogError(SC_ERR_MEM_ALLOC, "calloc() failed");
777  goto error;
778  }
779 
780  om->name = MODULE_NAME;
781  om->conf_name = script->val;
782  om->InitSubFunc = OutputLuaLogInitSub;
783  om->ThreadInit = LuaLogThreadInit;
784  om->ThreadDeinit = LuaLogThreadDeinit;
785 
786  if (opts.alproto == ALPROTO_HTTP && opts.streaming) {
787  om->StreamingLogFunc = LuaStreamingLogger;
789  om->alproto = ALPROTO_HTTP;
792  } else if (opts.alproto == ALPROTO_HTTP) {
793  om->TxLogFunc = LuaTxLogger;
794  om->alproto = ALPROTO_HTTP;
795  om->ts_log_progress = -1;
796  om->tc_log_progress = -1;
798  } else if (opts.alproto == ALPROTO_TLS) {
799  om->TxLogFunc = LuaTxLogger;
800  om->alproto = ALPROTO_TLS;
804  } else if (opts.alproto == ALPROTO_DNS) {
805  om->TxLogFunc = LuaTxLogger;
806  om->alproto = ALPROTO_DNS;
807  om->ts_log_progress = -1;
808  om->tc_log_progress = -1;
811  } else if (opts.alproto == ALPROTO_SSH) {
812  om->TxLogFunc = LuaTxLogger;
813  om->alproto = ALPROTO_SSH;
816  } else if (opts.alproto == ALPROTO_SMTP) {
817  om->TxLogFunc = LuaTxLogger;
818  om->alproto = ALPROTO_SMTP;
819  om->ts_log_progress = -1;
820  om->tc_log_progress = -1;
822  } else if (opts.packet && opts.alerts) {
823  om->PacketLogFunc = LuaPacketLoggerAlerts;
824  om->PacketConditionFunc = LuaPacketConditionAlerts;
825  } else if (opts.packet && opts.alerts == 0) {
826  om->PacketLogFunc = LuaPacketLogger;
827  om->PacketConditionFunc = LuaPacketCondition;
828  } else if (opts.file) {
829  om->FileLogFunc = LuaFileLogger;
831  } else if (opts.streaming && opts.tcp_data) {
832  om->StreamingLogFunc = LuaStreamingLogger;
834  } else if (opts.flow) {
835  om->FlowLogFunc = LuaFlowLogger;
836  } else if (opts.stats) {
837  om->StatsLogFunc = LuaStatsLogger;
838  } else {
839  SCLogError(SC_ERR_LUA_ERROR, "failed to setup thread module");
840  SCFree(om);
841  goto error;
842  }
843 
844  TAILQ_INSERT_TAIL(&output_ctx->submodules, om, entries);
845  }
846 
847  result.ctx = output_ctx;
848  result.ok = true;
849  return result;
850 
851 error:
852  if (output_ctx->DeInit)
853  output_ctx->DeInit(output_ctx);
854 
855  int failure_fatal = 0;
856  if (ConfGetBool("engine.init-failure-fatal", &failure_fatal) != 1) {
857  SCLogDebug("ConfGetBool could not load the value.");
858  }
859  if (failure_fatal) {
861  "Error during setup of lua output. Details should be "
862  "described in previous error messages. Shutting down...");
863  }
864 
865  return result;
866 }
867 
868 /** \internal
869  * \brief Run the scripts 'deinit' function
870  */
871 static void OutputLuaLogDoDeinit(LogLuaCtx *lua_ctx)
872 {
873  lua_State *luastate = lua_ctx->luastate;
874 
875  lua_getglobal(luastate, "deinit");
876  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
877  SCLogError(SC_ERR_LUA_ERROR, "no deinit function in script");
878  return;
879  }
880  //LuaPrintStack(luastate);
881 
882  if (lua_pcall(luastate, 0, 0, 0) != 0) {
883  SCLogError(SC_ERR_LUA_ERROR, "couldn't run script 'deinit' function: %s", lua_tostring(luastate, -1));
884  return;
885  }
886  LuaReturnState(luastate);
887 }
888 
889 /** \internal
890  * \brief Initialize the thread storage for lua
891  *
892  * Currently only stores a pointer to the global LogLuaCtx
893  */
894 static TmEcode LuaLogThreadInit(ThreadVars *t, const void *initdata, void **data)
895 {
896  LogLuaThreadCtx *td = SCMalloc(sizeof(*td));
897  if (unlikely(td == NULL))
898  return TM_ECODE_FAILED;
899  memset(td, 0, sizeof(*td));
900 
901  if (initdata == NULL) {
902  SCLogDebug("Error getting context for LuaLog. \"initdata\" argument NULL");
903  SCFree(td);
904  return TM_ECODE_FAILED;
905  }
906 
907  LogLuaCtx *lua_ctx = ((OutputCtx *)initdata)->data;
908  SCLogDebug("lua_ctx %p", lua_ctx);
909  td->lua_ctx = lua_ctx;
910  *data = (void *)td;
911  return TM_ECODE_OK;
912 }
913 
914 /** \internal
915  * \brief Deinit the thread storage for lua
916  *
917  * Calls OutputLuaLogDoDeinit if no-one else already did.
918  */
919 static TmEcode LuaLogThreadDeinit(ThreadVars *t, void *data)
920 {
921  LogLuaThreadCtx *td = (LogLuaThreadCtx *)data;
922  if (td == NULL) {
923  return TM_ECODE_OK;
924  }
925 
926  SCMutexLock(&td->lua_ctx->m);
927  if (td->lua_ctx->deinit_once == 0) {
928  OutputLuaLogDoDeinit(td->lua_ctx);
929  td->lua_ctx->deinit_once = 1;
930  }
931  SCMutexUnlock(&td->lua_ctx->m);
932 
933  /* clear memory */
934  memset(td, 0, sizeof(*td));
935 
936  SCFree(td);
937  return TM_ECODE_OK;
938 }
939 
940 void LuaLogRegister(void) {
941  /* register as separate module */
942  OutputRegisterModule(MODULE_NAME, "lua", OutputLuaLogInit);
943 }
944 
945 #else
946 
947 void LuaLogRegister (void) {
948  /* no-op */
949 }
950 
951 #endif
tm-threads.h
Packet_::proto
uint8_t proto
Definition: decode.h:433
AppLayerHtpNeedFileInspection
void AppLayerHtpNeedFileInspection(void)
Sets a flag that informs the HTP app layer that some module in the engine needs the http request file...
Definition: app-layer-htp.c:512
util-lua-ssh.h
PacketAlert_::s
const struct Signature_ * s
Definition: decode.h:276
app-layer-ssh.h
util-lua-hassh.h
util-lua-common.h
PACKET_ALERT_FLAG_TX
#define PACKET_ALERT_FLAG_TX
Definition: decode.h:289
TAILQ_INIT
#define TAILQ_INIT(head)
Definition: queue.h:370
ALPROTO_DNS
@ ALPROTO_DNS
Definition: app-layer-protos.h:41
PKT_IS_IPV6
#define PKT_IS_IPV6(p)
Definition: decode.h:255
ConfNode_::val
char * val
Definition: conf.h:34
ConfGetBool
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as an boolen.
Definition: conf.c:516
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
ALPROTO_TLS
@ ALPROTO_TLS
Definition: app-layer-protos.h:33
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:298
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
PacketAlerts_::alerts
PacketAlert alerts[PACKET_ALERT_MAX]
Definition: decode.h:297
Flow_::proto
uint8_t proto
Definition: flow.h:361
util-lua.h
AppProto
uint16_t AppProto
Definition: app-layer-protos.h:71
PacketAlerts_::cnt
uint16_t cnt
Definition: decode.h:296
threads.h
OutputModule_::name
const char * name
Definition: output.h:55
OutputModule_::StreamingLogFunc
StreamingLogger StreamingLogFunc
Definition: output.h:72
Flow_
Flow data structure.
Definition: flow.h:343
OutputModule_::FileLogFunc
FileLogger FileLogFunc
Definition: output.h:69
util-lua-ja3.h
OutputModule_::ts_log_progress
int ts_log_progress
Definition: output.h:77
TAILQ_FOREACH
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:350
StatsRecord_::value
uint64_t value
Definition: output-stats.h:32
OutputModule_::StatsLogFunc
StatsLogger StatsLogFunc
Definition: output.h:73
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
util-privs.h
TAILQ_INSERT_TAIL
#define TAILQ_INSERT_TAIL(head, elm, field)
Definition: queue.h:385
m
SCMutex m
Definition: flow-hash.h:3
OutputModule_::InitSubFunc
OutputInitSubFunc InitSubFunc
Definition: output.h:59
ALPROTO_SSH
@ ALPROTO_SSH
Definition: app-layer-protos.h:34
OutputModule_::PacketLogFunc
PacketLogger PacketLogFunc
Definition: output.h:65
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:79
OUTPUT_STREAMING_FLAG_TRANSACTION
#define OUTPUT_STREAMING_FLAG_TRANSACTION
Definition: output-streaming.h:36
OutputModule_::alproto
AppProto alproto
Definition: output.h:74
Packet_::alerts
PacketAlerts alerts
Definition: decode.h:558
util-unittest.h
OutputCtx_::data
void * data
Definition: tm-modules.h:81
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:78
PacketAlert_::tx_id
uint64_t tx_id
Definition: decode.h:277
OutputCtx_
Definition: tm-modules.h:78
OutputModule_::stream_type
enum OutputStreamingType stream_type
Definition: output.h:75
strlcpy
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
MODULE_NAME
#define MODULE_NAME
Definition: alert-debuglog.c:61
app-layer-htp.h
OutputModule_::ThreadInit
ThreadInitFunc ThreadInit
Definition: output.h:61
SSHTxLogCondition
int SSHTxLogCondition(ThreadVars *tv, const Packet *p, void *state, void *tx, uint64_t tx_id)
Definition: app-layer-ssh.c:74
util-debug.h
OutputInitResult_::ctx
OutputCtx * ctx
Definition: output.h:44
OutputModule_::ThreadDeinit
ThreadDeinitFunc ThreadDeinit
Definition: output.h:62
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
ALPROTO_SMTP
@ ALPROTO_SMTP
Definition: app-layer-protos.h:32
AppLayerHtpEnableRequestBodyCallback
void AppLayerHtpEnableRequestBodyCallback(void)
Sets a flag that informs the HTP app layer that some module in the engine needs the http request body...
Definition: app-layer-htp.c:470
AppLayerParserRegisterLogger
void AppLayerParserRegisterLogger(uint8_t ipproto, AppProto alproto)
Definition: app-layer-parser.c:455
OutputRegisterModule
void OutputRegisterModule(const char *, const char *, OutputInitFunc)
util-print.h
SCEnter
#define SCEnter(...)
Definition: util-debug.h:300
StatsRecord_::name
const char * name
Definition: output-stats.h:30
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:58
pkt-var.h
SC_ERR_LUA_ERROR
@ SC_ERR_LUA_ERROR
Definition: util-error.h:244
util-time.h
OutputInitResult_::ok
bool ok
Definition: output.h:45
app-layer-parser.h
TRUE
#define TRUE
Definition: suricata-common.h:33
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:282
StatsTable_
Definition: output-stats.h:36
FALSE
#define FALSE
Definition: suricata-common.h:34
OutputModule_::conf_name
const char * conf_name
Definition: output.h:56
OutputModule_::FlowLogFunc
FlowLogger FlowLogFunc
Definition: output.h:71
Packet_
Definition: decode.h:411
SC_ERR_SPRINTF
@ SC_ERR_SPRINTF
Definition: util-error.h:42
conf.h
TLS_HANDSHAKE_DONE
@ TLS_HANDSHAKE_DONE
Definition: app-layer-ssl.h:71
TmEcode
TmEcode
Definition: tm-threads-common.h:77
AppLayerHtpEnableResponseBodyCallback
void AppLayerHtpEnableResponseBodyCallback(void)
Sets a flag that informs the HTP app layer that some module in the engine needs the http request body...
Definition: app-layer-htp.c:483
CreateTimeString
void CreateTimeString(const struct timeval *ts, char *str, size_t size)
Definition: util-time.c:275
util-proto-name.h
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:217
TAILQ_FOREACH_SAFE
#define TAILQ_FOREACH_SAFE(var, head, field, tvar)
Definition: queue.h:356
AppLayerParserGetTx
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
Definition: app-layer-parser.c:1031
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
StatsTable_::stats
StatsRecord * stats
Definition: output-stats.h:37
util-lua-smtp.h
ConfNodeLookupChild
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:814
PacketAlert_::flags
uint8_t flags
Definition: decode.h:275
File_::flags
uint16_t flags
Definition: util-file.h:64
File_
Definition: util-file.h:63
OutputInitResult_
Definition: output.h:43
Packet_::flow
struct Flow_ * flow
Definition: decode.h:448
OutputModule_::TxLogCondition
TxLoggerCondition TxLogCondition
Definition: output.h:68
flags
uint8_t flags
Definition: decode-gre.h:0
Packet_::ts
struct timeval ts
Definition: decode.h:454
suricata-common.h
OutputCtx_::DeInit
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:84
STREAMING_HTTP_BODIES
@ STREAMING_HTTP_BODIES
Definition: output-streaming.h:40
StatsRecord_::tm_name
const char * tm_name
Definition: output-stats.h:31
SCLogError
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
Definition: util-debug.h:257
lua_State
void lua_State
Definition: suricata-common.h:509
util-lua-dns.h
FatalError
#define FatalError(x,...)
Definition: util-debug.h:532
LuaLogRegister
void LuaLogRegister(void)
Definition: output-lua.c:947
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:29
OutputModule_::PacketConditionFunc
PacketLogCondition PacketConditionFunc
Definition: output.h:66
threadvars.h
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
FILE_LOGGED
#define FILE_LOGGED
Definition: util-file.h:44
SCFree
#define SCFree(p)
Definition: util-mem.h:61
ConfNode_
Definition: conf.h:32
util-logopenfile.h
Flow_::alstate
void * alstate
Definition: flow.h:454
SC_ERR_FATAL
@ SC_ERR_FATAL
Definition: util-error.h:203
util-buffer.h
OutputModule_::TxLogFunc
TxLogger TxLogFunc
Definition: output.h:67
OutputModule_::tc_log_progress
int tc_log_progress
Definition: output.h:76
ALPROTO_HTTP
@ ALPROTO_HTTP
Definition: app-layer-protos.h:30
util-lua-tls.h
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
output-lua.h
SC_ERR_MEM_ALLOC
@ SC_ERR_MEM_ALLOC
Definition: util-error.h:31
PacketAlert_
Definition: decode.h:272
StatsRecord_::pvalue
uint64_t pvalue
Definition: output-stats.h:33
File_::txid
uint64_t txid
Definition: util-file.h:68
STREAMING_TCP_DATA
@ STREAMING_TCP_DATA
Definition: output-streaming.h:39
util-lua-http.h
OutputModule_
Definition: output.h:53
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:425
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:304
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
StatsTable_::nstats
uint32_t nstats
Definition: output-stats.h:39
PKT_IS_IPV4
#define PKT_IS_IPV4(p)
Definition: decode.h:254
SCMutex
#define SCMutex
Definition: threads-debug.h:114
app-layer-ssl.h
debug.h
output.h
app-layer.h
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:842