suricata
output-lua.c
Go to the documentation of this file.
1 /* Copyright (C) 2014-2022 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  */
24 
25 #include "suricata-common.h"
26 #include "output-lua.h"
27 
28 #ifdef HAVE_LUA
29 #include "util-print.h"
30 #include "util-unittest.h"
31 #include "util-debug.h"
32 #include "output.h"
33 #include "app-layer-htp.h"
34 #include "app-layer.h"
35 #include "app-layer-ssl.h"
36 #include "app-layer-ssh.h"
37 #include "app-layer-parser.h"
38 #include "util-privs.h"
39 #include "util-buffer.h"
40 #include "util-proto-name.h"
41 #include "util-logopenfile.h"
42 #include "util-time.h"
43 #include "util-lua.h"
44 #include "util-lua-common.h"
45 #include "util-lua-http.h"
46 #include "util-lua-dns.h"
47 #include "util-lua-ja3.h"
48 #include "util-lua-tls.h"
49 #include "util-lua-ssh.h"
50 #include "util-lua-hassh.h"
51 #include "util-lua-smtp.h"
52 
53 #define MODULE_NAME "LuaLog"
54 
55 /** \brief structure containing global config
56  * The OutputLuaLogInitSub which is run per script
57  * can access this to get global config info through
58  * it's parent_ctx->data ptr.
59  */
60 typedef struct LogLuaMasterCtx_ {
61  char path[PATH_MAX]; /**< contains script-dir */
62 } LogLuaMasterCtx;
63 
64 typedef struct LogLuaCtx_ {
65  SCMutex m;
66  lua_State *luastate;
67  int deinit_once;
68 } LogLuaCtx;
69 
70 typedef struct LogLuaThreadCtx_ {
71  LogLuaCtx *lua_ctx;
72 } LogLuaThreadCtx;
73 
74 static TmEcode LuaLogThreadInit(ThreadVars *t, const void *initdata, void **data);
75 static TmEcode LuaLogThreadDeinit(ThreadVars *t, void *data);
76 
77 /** \internal
78  * \brief TX logger for lua scripts
79  *
80  * A single call to this function will run one script on a single
81  * transaction.
82  *
83  * NOTE: The flow (f) also referenced by p->flow is locked.
84  */
85 static int LuaTxLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *alstate, void *txptr, uint64_t tx_id)
86 {
87  SCEnter();
88 
89  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
90 
91  SCMutexLock(&td->lua_ctx->m);
92 
93  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
94  LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
95  LuaStateSetTX(td->lua_ctx->luastate, txptr, tx_id);
96  LuaStateSetFlow(td->lua_ctx->luastate, f);
97 
98  /* prepare data to pass to script */
99  lua_getglobal(td->lua_ctx->luastate, "log");
100  lua_newtable(td->lua_ctx->luastate);
101  LuaPushTableKeyValueInt(td->lua_ctx->luastate, "tx_id", (int)(tx_id));
102 
103  int retval = lua_pcall(td->lua_ctx->luastate, 1, 0, 0);
104  if (retval != 0) {
105  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
106  }
107 
108  SCMutexUnlock(&td->lua_ctx->m);
109  SCReturnInt(0);
110 }
111 
112 /** \internal
113  * \brief Streaming logger for lua scripts
114  *
115  * Hooks into the Streaming Logger API. Gets called for each chunk of new
116  * streaming data.
117  */
118 static int LuaStreamingLogger(ThreadVars *tv, void *thread_data, const Flow *f,
119  const uint8_t *data, uint32_t data_len, uint64_t tx_id, uint8_t flags)
120 {
121  SCEnter();
122 
123  void *txptr = NULL;
124  LuaStreamingBuffer b = { data, data_len, flags };
125 
126  SCLogDebug("flags %02x", flags);
127 
129  if (f && f->alstate)
130  txptr = AppLayerParserGetTx(f->proto, f->alproto, f->alstate, tx_id);
131  }
132 
133  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
134 
135  SCMutexLock(&td->lua_ctx->m);
136 
137  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
139  LuaStateSetTX(td->lua_ctx->luastate, txptr, tx_id);
140  LuaStateSetFlow(td->lua_ctx->luastate, (Flow *)f);
141  LuaStateSetStreamingBuffer(td->lua_ctx->luastate, &b);
142 
143  /* prepare data to pass to script */
144  lua_getglobal(td->lua_ctx->luastate, "log");
145  lua_newtable(td->lua_ctx->luastate);
146 
148  LuaPushTableKeyValueInt(td->lua_ctx->luastate, "tx_id", (int)(tx_id));
149 
150  int retval = lua_pcall(td->lua_ctx->luastate, 1, 0, 0);
151  if (retval != 0) {
152  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
153  }
154 
155  SCMutexUnlock(&td->lua_ctx->m);
156 
158 }
159 
160 /** \internal
161  * \brief Packet Logger for lua scripts, for alerts
162  *
163  * A single call to this function will run one script for a single
164  * packet. If it is called, it means that the registered condition
165  * function has returned TRUE.
166  *
167  * The script is called once for each alert stored in the packet.
168  *
169  * NOTE: p->flow is UNlocked
170  */
171 static int LuaPacketLoggerAlerts(ThreadVars *tv, void *thread_data, const Packet *p)
172 {
173  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
174 
175  char timebuf[64];
176  CreateTimeString(p->ts, timebuf, sizeof(timebuf));
177 
178  if (!(PKT_IS_IPV4(p)) && !(PKT_IS_IPV6(p))) {
179  /* decoder event */
180  goto not_supported;
181  }
182 
183  /* loop through alerts stored in the packet */
184  SCMutexLock(&td->lua_ctx->m);
185  uint16_t cnt;
186  for (cnt = 0; cnt < p->alerts.cnt; cnt++) {
187  const PacketAlert *pa = &p->alerts.alerts[cnt];
188  if (unlikely(pa->s == NULL)) {
189  continue;
190  }
191 
192  lua_getglobal(td->lua_ctx->luastate, "log");
193 
194  void *txptr = NULL;
195  if (p->flow && p->flow->alstate && (pa->flags & PACKET_ALERT_FLAG_TX))
196  txptr = AppLayerParserGetTx(
197  p->flow->proto, p->flow->alproto, p->flow->alstate, pa->tx_id);
198 
199  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
200  LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
201  LuaStateSetTX(td->lua_ctx->luastate, txptr, pa->tx_id);
202  LuaStateSetFlow(td->lua_ctx->luastate, p->flow);
203  LuaStateSetPacketAlert(td->lua_ctx->luastate, (PacketAlert *)pa);
204 
205  /* prepare data to pass to script */
206  //lua_newtable(td->lua_ctx->luastate);
207 
208  int retval = lua_pcall(td->lua_ctx->luastate, 0, 0, 0);
209  if (retval != 0) {
210  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
211  }
212  }
213  SCMutexUnlock(&td->lua_ctx->m);
214 not_supported:
215  SCReturnInt(0);
216 }
217 
218 static int LuaPacketConditionAlerts(ThreadVars *tv, void *data, const Packet *p)
219 {
220  if (p->alerts.cnt > 0)
221  return TRUE;
222  return FALSE;
223 }
224 
225 /** \internal
226  * \brief Packet Logger for lua scripts, for packets
227  *
228  * A single call to this function will run one script for a single
229  * packet. If it is called, it means that the registered condition
230  * function has returned TRUE.
231  *
232  * The script is called once for each packet.
233  *
234  * NOTE: p->flow is UNlocked
235  */
236 static int LuaPacketLogger(ThreadVars *tv, void *thread_data, const Packet *p)
237 {
238  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
239 
240  char timebuf[64];
241 
242  if ((!(PKT_IS_IPV4(p))) && (!(PKT_IS_IPV6(p)))) {
243  goto not_supported;
244  }
245 
246  CreateTimeString(p->ts, timebuf, sizeof(timebuf));
247 
248  /* loop through alerts stored in the packet */
249  SCMutexLock(&td->lua_ctx->m);
250  lua_getglobal(td->lua_ctx->luastate, "log");
251 
252  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
253  LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
254  LuaStateSetFlow(td->lua_ctx->luastate, p->flow);
255 
256  /* prepare data to pass to script */
257  lua_newtable(td->lua_ctx->luastate);
258 
259  int retval = lua_pcall(td->lua_ctx->luastate, 1, 0, 0);
260  if (retval != 0) {
261  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
262  }
263  SCMutexUnlock(&td->lua_ctx->m);
264 not_supported:
265  SCReturnInt(0);
266 }
267 
268 static int LuaPacketCondition(ThreadVars *tv, void *data, const Packet *p)
269 {
270  return TRUE;
271 }
272 
273 /** \internal
274  * \brief File API Logger function for Lua scripts
275  *
276  * Executes a script once for one file.
277  *
278  * NOTE p->flow is locked at this point
279  */
280 static int LuaFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff,
281  void *tx, const uint64_t tx_id, uint8_t dir)
282 {
283  SCEnter();
284  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
285 
286  if ((!(PKT_IS_IPV4(p))) && (!(PKT_IS_IPV6(p))))
287  return 0;
288 
289  BUG_ON(ff->flags & FILE_LOGGED);
290 
291  SCLogDebug("ff %p", ff);
292 
293  SCMutexLock(&td->lua_ctx->m);
294 
295  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
296  LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
297  LuaStateSetTX(td->lua_ctx->luastate, tx, tx_id);
298  LuaStateSetFlow(td->lua_ctx->luastate, p->flow);
299  LuaStateSetFile(td->lua_ctx->luastate, (File *)ff);
300 
301  /* get the lua function to call */
302  lua_getglobal(td->lua_ctx->luastate, "log");
303 
304  int retval = lua_pcall(td->lua_ctx->luastate, 0, 0, 0);
305  if (retval != 0) {
306  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
307  }
308  SCMutexUnlock(&td->lua_ctx->m);
309  return 0;
310 }
311 
312 /** \internal
313  * \brief Flow API Logger function for Lua scripts
314  *
315  * Executes a script once for one flow
316  *
317  * Note: flow 'f' is locked
318  */
319 static int LuaFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
320 {
321  SCEnter();
322  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
323 
324  SCLogDebug("f %p", f);
325 
326  SCMutexLock(&td->lua_ctx->m);
327 
328  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
329  LuaStateSetFlow(td->lua_ctx->luastate, f);
330 
331  /* get the lua function to call */
332  lua_getglobal(td->lua_ctx->luastate, "log");
333 
334  int retval = lua_pcall(td->lua_ctx->luastate, 0, 0, 0);
335  if (retval != 0) {
336  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
337  }
338  SCMutexUnlock(&td->lua_ctx->m);
339  return 0;
340 }
341 
342 
343 
344 static int LuaStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable *st)
345 {
346  SCEnter();
347  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
348 
349  SCMutexLock(&td->lua_ctx->m);
350 
351  lua_State *luastate = td->lua_ctx->luastate;
352  /* get the lua function to call */
353  lua_getglobal(td->lua_ctx->luastate, "log");
354 
355  /* create lua array, which is really just a table. The key is an int (1-x),
356  * the value another table with named fields: name, tm_name, value, pvalue.
357  * { 1, { name=<name>, tmname=<tm_name>, value=<value>, pvalue=<pvalue>}}
358  * { 2, { name=<name>, tmname=<tm_name>, value=<value>, pvalue=<pvalue>}}
359  * etc
360  */
361  lua_newtable(luastate);
362  uint32_t u = 0;
363  for (; u < st->nstats; u++) {
364  lua_pushinteger(luastate, u + 1);
365 
366  lua_newtable(luastate);
367 
368  lua_pushstring(luastate, "name");
369  lua_pushstring(luastate, st->stats[u].name);
370  lua_settable(luastate, -3);
371 
372  lua_pushstring(luastate, "tmname");
373  lua_pushstring(luastate, st->stats[u].tm_name);
374  lua_settable(luastate, -3);
375 
376  lua_pushstring(luastate, "value");
377  lua_pushinteger(luastate, st->stats[u].value);
378  lua_settable(luastate, -3);
379 
380  lua_pushstring(luastate, "pvalue");
381  lua_pushinteger(luastate, st->stats[u].pvalue);
382  lua_settable(luastate, -3);
383 
384  lua_settable(luastate, -3);
385  }
386 
387  int retval = lua_pcall(td->lua_ctx->luastate, 1, 0, 0);
388  if (retval != 0) {
389  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
390  }
391  SCMutexUnlock(&td->lua_ctx->m);
392  return 0;
393 
394 }
395 
396 typedef struct LogLuaScriptOptions_ {
397  AppProto alproto;
398  int packet;
399  int alerts;
400  int file;
401  int streaming;
402  int tcp_data;
403  int http_body;
404  int flow;
405  int stats;
406 } LogLuaScriptOptions;
407 
408 /** \brief load and evaluate the script
409  *
410  * This function parses the script, checks if all the required functions
411  * are defined and runs the 'init' function. The init function will inform
412  * us what the scripts needs are.
413  *
414  * \param filename filename of lua script file
415  * \param options struct to pass script requirements/options back to caller
416  * \retval errcode 0 ok, -1 error
417  */
418 static int LuaScriptInit(const char *filename, LogLuaScriptOptions *options) {
419  lua_State *luastate = LuaGetState();
420  if (luastate == NULL)
421  goto error;
422  luaL_openlibs(luastate);
423 
424  int status = luaL_loadfile(luastate, filename);
425  if (status) {
426  SCLogError("couldn't load file: %s", lua_tostring(luastate, -1));
427  goto error;
428  }
429 
430  /* prime the script (or something) */
431  if (lua_pcall(luastate, 0, 0, 0) != 0) {
432  SCLogError("couldn't prime file: %s", lua_tostring(luastate, -1));
433  goto error;
434  }
435 
436  lua_getglobal(luastate, "init");
437  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
438  SCLogError("no init function in script");
439  goto error;
440  }
441 
442  lua_newtable(luastate); /* stack at -1 */
443  if (lua_gettop(luastate) == 0 || lua_type(luastate, 2) != LUA_TTABLE) {
444  SCLogError("no table setup");
445  goto error;
446  }
447 
448  lua_pushliteral(luastate, "script_api_ver");
449  lua_pushnumber (luastate, 1);
450  lua_settable(luastate, -3);
451 
452  if (lua_pcall(luastate, 1, 1, 0) != 0) {
453  SCLogError("couldn't run script 'init' function: %s", lua_tostring(luastate, -1));
454  goto error;
455  }
456 
457  /* process returns from script */
458  if (lua_gettop(luastate) == 0) {
459  SCLogError("init function in script should return table, nothing returned");
460  goto error;
461  }
462  if (lua_type(luastate, 1) != LUA_TTABLE) {
463  SCLogError("init function in script should return table, returned is not table");
464  goto error;
465  }
466 
467  lua_pushnil(luastate);
468  const char *k, *v;
469  while (lua_next(luastate, -2)) {
470  k = lua_tostring(luastate, -2);
471  if (k == NULL)
472  continue;
473 
474  v = lua_tostring(luastate, -1);
475  lua_pop(luastate, 1);
476  if (v == NULL)
477  continue;
478 
479  SCLogDebug("k='%s', v='%s'", k, v);
480 
481  if (strcmp(k,"protocol") == 0 && strcmp(v, "http") == 0)
482  options->alproto = ALPROTO_HTTP1;
483  else if (strcmp(k,"protocol") == 0 && strcmp(v, "dns") == 0)
484  options->alproto = ALPROTO_DNS;
485  else if (strcmp(k,"protocol") == 0 && strcmp(v, "tls") == 0)
486  options->alproto = ALPROTO_TLS;
487  else if (strcmp(k,"protocol") == 0 && strcmp(v, "ssh") == 0)
488  options->alproto = ALPROTO_SSH;
489  else if (strcmp(k,"protocol") == 0 && strcmp(v, "smtp") == 0)
490  options->alproto = ALPROTO_SMTP;
491  else if (strcmp(k, "type") == 0 && strcmp(v, "packet") == 0)
492  options->packet = 1;
493  else if (strcmp(k, "filter") == 0 && strcmp(v, "alerts") == 0)
494  options->alerts = 1;
495  else if (strcmp(k, "type") == 0 && strcmp(v, "file") == 0)
496  options->file = 1;
497  else if (strcmp(k, "type") == 0 && strcmp(v, "streaming") == 0)
498  options->streaming = 1;
499  else if (strcmp(k, "type") == 0 && strcmp(v, "flow") == 0)
500  options->flow = 1;
501  else if (strcmp(k, "filter") == 0 && strcmp(v, "tcp") == 0)
502  options->tcp_data = 1;
503  else if (strcmp(k, "type") == 0 && strcmp(v, "stats") == 0)
504  options->stats = 1;
505  else
506  SCLogInfo("unknown key and/or value: k='%s', v='%s'", k, v);
507  }
508 
509  if (((options->alproto != ALPROTO_UNKNOWN)) + options->packet + options->file > 1) {
510  SCLogError("invalid combination of 'needs' in the script");
511  goto error;
512  }
513 
514  lua_getglobal(luastate, "setup");
515  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
516  SCLogError("no setup function in script");
517  goto error;
518  }
519 
520  lua_getglobal(luastate, "log");
521  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
522  SCLogError("no log function in script");
523  goto error;
524  }
525 
526  lua_getglobal(luastate, "deinit");
527  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
528  SCLogError("no deinit function in script");
529  goto error;
530  }
531 
532  LuaReturnState(luastate);
533  return 0;
534 error:
535  if (luastate)
536  LuaReturnState(luastate);
537  return -1;
538 }
539 
540 /** \brief setup a luastate for use at runtime
541  *
542  * This loads the script, primes it and then runs the 'setup' function.
543  *
544  * \retval state Returns the set up luastate on success, NULL on error
545  */
546 static lua_State *LuaScriptSetup(const char *filename)
547 {
548  lua_State *luastate = LuaGetState();
549  if (luastate == NULL) {
550  SCLogError("luaL_newstate failed");
551  goto error;
552  }
553 
554  luaL_openlibs(luastate);
555 
556  int status = luaL_loadfile(luastate, filename);
557  if (status) {
558  SCLogError("couldn't load file: %s", lua_tostring(luastate, -1));
559  goto error;
560  }
561 
562  /* prime the script */
563  if (lua_pcall(luastate, 0, 0, 0) != 0) {
564  SCLogError("couldn't prime file: %s", lua_tostring(luastate, -1));
565  goto error;
566  }
567 
568  lua_getglobal(luastate, "setup");
569 
570  /* register functions common to all */
571  LuaRegisterFunctions(luastate);
572  /* unconditionally register http function. They will only work
573  * if the tx is registered in the state at runtime though. */
574  LuaRegisterHttpFunctions(luastate);
575  LuaRegisterDnsFunctions(luastate);
576  LuaRegisterJa3Functions(luastate);
577  LuaRegisterTlsFunctions(luastate);
578  LuaRegisterSshFunctions(luastate);
579  LuaRegisterHasshFunctions(luastate);
580  LuaRegisterSmtpFunctions(luastate);
581 
582  if (lua_pcall(luastate, 0, 0, 0) != 0) {
583  SCLogError("couldn't run script 'setup' function: %s", lua_tostring(luastate, -1));
584  goto error;
585  }
586 
587  SCLogDebug("lua_State %p is set up", luastate);
588  return luastate;
589 error:
590  if (luastate)
591  LuaReturnState(luastate);
592  return NULL;
593 }
594 
595 static void LogLuaSubFree(OutputCtx *oc) {
596  if (oc->data)
597  SCFree(oc->data);
598  SCFree(oc);
599 }
600 
601 /** \brief initialize output for a script instance
602  *
603  * Runs script 'setup' function.
604  */
605 static OutputInitResult OutputLuaLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
606 {
607  OutputInitResult result = { NULL, false };
608  if (conf == NULL)
609  return result;
610 
611  LogLuaCtx *lua_ctx = SCMalloc(sizeof(LogLuaCtx));
612  if (unlikely(lua_ctx == NULL))
613  return result;
614  memset(lua_ctx, 0x00, sizeof(*lua_ctx));
615 
616  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
617  if (unlikely(output_ctx == NULL)) {
618  SCFree(lua_ctx);
619  return result;
620  }
621 
622  SCMutexInit(&lua_ctx->m, NULL);
623 
624  const char *dir = "";
625  if (parent_ctx && parent_ctx->data) {
626  LogLuaMasterCtx *mc = parent_ctx->data;
627  dir = mc->path;
628  }
629 
630  char path[PATH_MAX] = "";
631  int ret = snprintf(path, sizeof(path),"%s%s%s", dir, strlen(dir) ? "/" : "", conf->val);
632  if (ret < 0 || ret == sizeof(path)) {
633  SCLogError("failed to construct lua script path");
634  goto error;
635  }
636  SCLogDebug("script full path %s", path);
637 
638  SCMutexLock(&lua_ctx->m);
639  lua_ctx->luastate = LuaScriptSetup(path);
640  SCMutexUnlock(&lua_ctx->m);
641  if (lua_ctx->luastate == NULL)
642  goto error;
643 
644  SCLogDebug("lua_ctx %p", lua_ctx);
645 
646  output_ctx->data = lua_ctx;
647  output_ctx->DeInit = LogLuaSubFree;
648 
649  result.ctx = output_ctx;
650  result.ok = true;
651  return result;
652 error:
653  SCMutexDestroy(&lua_ctx->m);
654  SCFree(lua_ctx);
655  SCFree(output_ctx);
656  return result;
657 }
658 
659 static void LogLuaMasterFree(OutputCtx *oc)
660 {
661  if (oc->data)
662  SCFree(oc->data);
663 
664  OutputModule *om, *tom;
665  TAILQ_FOREACH_SAFE(om, &oc->submodules, entries, tom) {
666  SCFree(om);
667  }
668  SCFree(oc);
669 }
670 
671 /** \internal
672  * \brief initialize output instance for lua module
673  *
674  * Parses nested script list, primes them to find out what they
675  * inspect, then fills the OutputCtx::submodules list with the
676  * proper Logger function for the data type the script needs.
677  */
678 static OutputInitResult OutputLuaLogInit(ConfNode *conf)
679 {
680  OutputInitResult result = { NULL, false };
681  const char *dir = ConfNodeLookupChildValue(conf, "scripts-dir");
682  if (dir == NULL)
683  dir = "";
684 
685  ConfNode *scripts = ConfNodeLookupChild(conf, "scripts");
686  if (scripts == NULL) {
687  /* No "outputs" section in the configuration. */
688  SCLogInfo("scripts not defined");
689  return result;
690  }
691 
692  /* global output ctx setup */
693  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
694  if (unlikely(output_ctx == NULL)) {
695  return result;
696  }
697  output_ctx->DeInit = LogLuaMasterFree;
698  output_ctx->data = SCCalloc(1, sizeof(LogLuaMasterCtx));
699  if (unlikely(output_ctx->data == NULL)) {
700  SCFree(output_ctx);
701  return result;
702  }
703  LogLuaMasterCtx *master_config = output_ctx->data;
704  strlcpy(master_config->path, dir, sizeof(master_config->path));
705  TAILQ_INIT(&output_ctx->submodules);
706 
707  /* check the enables scripts and set them up as submodules */
708  ConfNode *script;
709  TAILQ_FOREACH(script, &scripts->head, next) {
710  SCLogInfo("enabling script %s", script->val);
711  LogLuaScriptOptions opts;
712  memset(&opts, 0x00, sizeof(opts));
713 
714  char path[PATH_MAX] = "";
715  snprintf(path, sizeof(path),"%s%s%s", dir, strlen(dir) ? "/" : "", script->val);
716  SCLogDebug("script full path %s", path);
717 
718  int r = LuaScriptInit(path, &opts);
719  if (r != 0) {
720  SCLogError("couldn't initialize script");
721  goto error;
722  }
723 
724  /* create an OutputModule for this script, based
725  * on it's needs. */
726  OutputModule *om = SCCalloc(1, sizeof(*om));
727  if (om == NULL) {
728  SCLogError("calloc() failed");
729  goto error;
730  }
731 
732  om->name = MODULE_NAME;
733  om->conf_name = script->val;
734  om->InitSubFunc = OutputLuaLogInitSub;
735  om->ThreadInit = LuaLogThreadInit;
736  om->ThreadDeinit = LuaLogThreadDeinit;
737 
738  if (opts.alproto == ALPROTO_HTTP1 && opts.streaming) {
739  om->StreamingLogFunc = LuaStreamingLogger;
741  om->alproto = ALPROTO_HTTP1;
744  } else if (opts.alproto == ALPROTO_HTTP1) {
745  om->TxLogFunc = LuaTxLogger;
746  om->alproto = ALPROTO_HTTP1;
747  om->ts_log_progress = -1;
748  om->tc_log_progress = -1;
750  } else if (opts.alproto == ALPROTO_TLS) {
751  om->TxLogFunc = LuaTxLogger;
752  om->alproto = ALPROTO_TLS;
756  } else if (opts.alproto == ALPROTO_DNS) {
757  om->TxLogFunc = LuaTxLogger;
758  om->alproto = ALPROTO_DNS;
759  om->ts_log_progress = -1;
760  om->tc_log_progress = -1;
763  } else if (opts.alproto == ALPROTO_SSH) {
764  om->TxLogFunc = LuaTxLogger;
765  om->alproto = ALPROTO_SSH;
768  } else if (opts.alproto == ALPROTO_SMTP) {
769  om->TxLogFunc = LuaTxLogger;
770  om->alproto = ALPROTO_SMTP;
771  om->ts_log_progress = -1;
772  om->tc_log_progress = -1;
774  } else if (opts.packet && opts.alerts) {
775  om->PacketLogFunc = LuaPacketLoggerAlerts;
776  om->PacketConditionFunc = LuaPacketConditionAlerts;
777  } else if (opts.packet && opts.alerts == 0) {
778  om->PacketLogFunc = LuaPacketLogger;
779  om->PacketConditionFunc = LuaPacketCondition;
780  } else if (opts.file) {
781  om->FileLogFunc = LuaFileLogger;
783  } else if (opts.streaming && opts.tcp_data) {
784  om->StreamingLogFunc = LuaStreamingLogger;
786  } else if (opts.flow) {
787  om->FlowLogFunc = LuaFlowLogger;
788  } else if (opts.stats) {
789  om->StatsLogFunc = LuaStatsLogger;
790  } else {
791  SCLogError("failed to setup thread module");
792  SCFree(om);
793  goto error;
794  }
795 
796  TAILQ_INSERT_TAIL(&output_ctx->submodules, om, entries);
797  }
798 
799  result.ctx = output_ctx;
800  result.ok = true;
801  return result;
802 
803 error:
804  if (output_ctx->DeInit)
805  output_ctx->DeInit(output_ctx);
806 
807  int failure_fatal = 0;
808  if (ConfGetBool("engine.init-failure-fatal", &failure_fatal) != 1) {
809  SCLogDebug("ConfGetBool could not load the value.");
810  }
811  if (failure_fatal) {
812  FatalError("Error during setup of lua output. Details should be "
813  "described in previous error messages. Shutting down...");
814  }
815 
816  return result;
817 }
818 
819 /** \internal
820  * \brief Run the scripts 'deinit' function
821  */
822 static void OutputLuaLogDoDeinit(LogLuaCtx *lua_ctx)
823 {
824  lua_State *luastate = lua_ctx->luastate;
825 
826  lua_getglobal(luastate, "deinit");
827  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
828  SCLogError("no deinit function in script");
829  return;
830  }
831  //LuaPrintStack(luastate);
832 
833  if (lua_pcall(luastate, 0, 0, 0) != 0) {
834  SCLogError("couldn't run script 'deinit' function: %s", lua_tostring(luastate, -1));
835  return;
836  }
837  LuaReturnState(luastate);
838 }
839 
840 /** \internal
841  * \brief Initialize the thread storage for lua
842  *
843  * Currently only stores a pointer to the global LogLuaCtx
844  */
845 static TmEcode LuaLogThreadInit(ThreadVars *t, const void *initdata, void **data)
846 {
847  LogLuaThreadCtx *td = SCMalloc(sizeof(*td));
848  if (unlikely(td == NULL))
849  return TM_ECODE_FAILED;
850  memset(td, 0, sizeof(*td));
851 
852  if (initdata == NULL) {
853  SCLogDebug("Error getting context for LuaLog. \"initdata\" argument NULL");
854  SCFree(td);
855  return TM_ECODE_FAILED;
856  }
857 
858  LogLuaCtx *lua_ctx = ((OutputCtx *)initdata)->data;
859  SCLogDebug("lua_ctx %p", lua_ctx);
860  td->lua_ctx = lua_ctx;
861  *data = (void *)td;
862  return TM_ECODE_OK;
863 }
864 
865 /** \internal
866  * \brief Deinit the thread storage for lua
867  *
868  * Calls OutputLuaLogDoDeinit if no-one else already did.
869  */
870 static TmEcode LuaLogThreadDeinit(ThreadVars *t, void *data)
871 {
872  LogLuaThreadCtx *td = (LogLuaThreadCtx *)data;
873  if (td == NULL) {
874  return TM_ECODE_OK;
875  }
876 
877  SCMutexLock(&td->lua_ctx->m);
878  if (td->lua_ctx->deinit_once == 0) {
879  OutputLuaLogDoDeinit(td->lua_ctx);
880  td->lua_ctx->deinit_once = 1;
881  }
882  SCMutexUnlock(&td->lua_ctx->m);
883 
884  /* clear memory */
885  memset(td, 0, sizeof(*td));
886 
887  SCFree(td);
888  return TM_ECODE_OK;
889 }
890 
891 void LuaLogRegister(void) {
892  /* register as separate module */
893  OutputRegisterModule(MODULE_NAME, "lua", OutputLuaLogInit);
894 }
895 
896 #else /* HAVE_LUA */
897 
898 void LuaLogRegister (void) {
899  /* no-op */
900 }
901 
902 #endif /* HAVE_LUA */
AppLayerHtpNeedFileInspection
void AppLayerHtpNeedFileInspection(void)
Sets a flag that informs the HTP app layer that some module in the engine needs the http request file...
Definition: app-layer-htp.c:512
util-lua-ssh.h
PacketAlert_::s
const struct Signature_ * s
Definition: decode.h:267
app-layer-ssh.h
util-lua-hassh.h
util-lua-common.h
PACKET_ALERT_FLAG_TX
#define PACKET_ALERT_FLAG_TX
Definition: decode.h:279
TAILQ_INIT
#define TAILQ_INIT(head)
Definition: queue.h:262
ALPROTO_DNS
@ ALPROTO_DNS
Definition: app-layer-protos.h:41
PKT_IS_IPV6
#define PKT_IS_IPV6(p)
Definition: decode.h:246
ConfNode_::val
char * val
Definition: conf.h:34
ConfGetBool
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as a boolean.
Definition: conf.c:483
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
ALPROTO_TLS
@ ALPROTO_TLS
Definition: app-layer-protos.h:33
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
Flow_::proto
uint8_t proto
Definition: flow.h:369
util-lua.h
AppProto
uint16_t AppProto
Definition: app-layer-protos.h:80
PacketAlerts_::cnt
uint16_t cnt
Definition: decode.h:289
OutputModule_::name
const char * name
Definition: output.h:58
OutputModule_::StreamingLogFunc
StreamingLogger StreamingLogFunc
Definition: output.h:75
Flow_
Flow data structure.
Definition: flow.h:347
OutputModule_::FileLogFunc
FileLogger FileLogFunc
Definition: output.h:72
util-lua-ja3.h
OutputModule_::ts_log_progress
int ts_log_progress
Definition: output.h:80
TAILQ_FOREACH
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:252
PacketAlerts_::alerts
PacketAlert * alerts
Definition: decode.h:292
OutputModule_::StatsLogFunc
StatsLogger StatsLogFunc
Definition: output.h:76
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
util-privs.h
TAILQ_INSERT_TAIL
#define TAILQ_INSERT_TAIL(head, elm, field)
Definition: queue.h:294
m
SCMutex m
Definition: flow-hash.h:6
OutputModule_::InitSubFunc
OutputInitSubFunc InitSubFunc
Definition: output.h:62
ALPROTO_SSH
@ ALPROTO_SSH
Definition: app-layer-protos.h:34
StatsRecord_::pvalue
int64_t pvalue
Definition: output-stats.h:36
OutputModule_::PacketLogFunc
PacketLogger PacketLogFunc
Definition: output.h:68
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:85
OUTPUT_STREAMING_FLAG_TRANSACTION
#define OUTPUT_STREAMING_FLAG_TRANSACTION
Definition: output-streaming.h:34
OutputModule_::alproto
AppProto alproto
Definition: output.h:77
Packet_::alerts
PacketAlerts alerts
Definition: decode.h:592
util-unittest.h
OutputCtx_::data
void * data
Definition: tm-modules.h:87
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:84
PacketAlert_::tx_id
uint64_t tx_id
Definition: decode.h:268
OutputCtx_
Definition: tm-modules.h:84
OutputModule_::stream_type
enum OutputStreamingType stream_type
Definition: output.h:78
strlcpy
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
MODULE_NAME
#define MODULE_NAME
Definition: alert-debuglog.c:61
app-layer-htp.h
OutputModule_::ThreadInit
ThreadInitFunc ThreadInit
Definition: output.h:64
SSHTxLogCondition
int SSHTxLogCondition(ThreadVars *tv, const Packet *p, void *state, void *tx, uint64_t tx_id)
Definition: app-layer-ssh.c:74
util-debug.h
OutputInitResult_::ctx
OutputCtx * ctx
Definition: output.h:47
OutputModule_::ThreadDeinit
ThreadDeinitFunc ThreadDeinit
Definition: output.h:65
Packet_::ts
SCTime_t ts
Definition: decode.h:475
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
ALPROTO_SMTP
@ ALPROTO_SMTP
Definition: app-layer-protos.h:32
TLS_HANDSHAKE_DONE
@ TLS_HANDSHAKE_DONE
Definition: app-layer-ssl.h:79
AppLayerHtpEnableRequestBodyCallback
void AppLayerHtpEnableRequestBodyCallback(void)
Sets a flag that informs the HTP app layer that some module in the engine needs the http request body...
Definition: app-layer-htp.c:470
AppLayerParserRegisterLogger
void AppLayerParserRegisterLogger(uint8_t ipproto, AppProto alproto)
Definition: app-layer-parser.c:477
OutputRegisterModule
void OutputRegisterModule(const char *, const char *, OutputInitFunc)
util-print.h
SCEnter
#define SCEnter(...)
Definition: util-debug.h:271
StatsRecord_::name
const char * name
Definition: output-stats.h:32
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:57
util-time.h
OutputInitResult_::ok
bool ok
Definition: output.h:48
app-layer-parser.h
TRUE
#define TRUE
Definition: suricata-common.h:33
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:295
StatsTable_
Definition: output-stats.h:39
FALSE
#define FALSE
Definition: suricata-common.h:34
StatsRecord_::value
int64_t value
Definition: output-stats.h:35
OutputModule_::conf_name
const char * conf_name
Definition: output.h:59
OutputModule_::FlowLogFunc
FlowLogger FlowLogFunc
Definition: output.h:74
Packet_
Definition: decode.h:430
TmEcode
TmEcode
Definition: tm-threads-common.h:83
AppLayerHtpEnableResponseBodyCallback
void AppLayerHtpEnableResponseBodyCallback(void)
Sets a flag that informs the HTP app layer that some module in the engine needs the http request body...
Definition: app-layer-htp.c:483
util-proto-name.h
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:224
TAILQ_FOREACH_SAFE
#define TAILQ_FOREACH_SAFE(var, head, field, tvar)
Definition: queue.h:329
AppLayerParserGetTx
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
Definition: app-layer-parser.c:1127
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
StatsTable_::stats
StatsRecord * stats
Definition: output-stats.h:40
util-lua-smtp.h
ConfNodeLookupChild
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:786
PacketAlert_::flags
uint8_t flags
Definition: decode.h:266
File_::flags
uint16_t flags
Definition: util-file.h:80
File_
Definition: util-file.h:79
OutputInitResult_
Definition: output.h:46
Packet_::flow
struct Flow_ * flow
Definition: decode.h:469
OutputModule_::TxLogCondition
TxLoggerCondition TxLogCondition
Definition: output.h:71
flags
uint8_t flags
Definition: decode-gre.h:0
suricata-common.h
OutputCtx_::DeInit
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:90
STREAMING_HTTP_BODIES
@ STREAMING_HTTP_BODIES
Definition: output-streaming.h:38
ALPROTO_HTTP1
@ ALPROTO_HTTP1
Definition: app-layer-protos.h:30
StatsRecord_::tm_name
const char * tm_name
Definition: output-stats.h:34
lua_State
void lua_State
Definition: suricata-common.h:495
util-lua-dns.h
FatalError
#define FatalError(...)
Definition: util-debug.h:502
LuaLogRegister
void LuaLogRegister(void)
Definition: output-lua.c:898
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
OutputModule_::PacketConditionFunc
PacketLogCondition PacketConditionFunc
Definition: output.h:69
SCMalloc
#define SCMalloc(sz)
Definition: util-mem.h:47
FILE_LOGGED
#define FILE_LOGGED
Definition: util-file.h:53
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:261
SCFree
#define SCFree(p)
Definition: util-mem.h:61
ConfNode_
Definition: conf.h:32
util-logopenfile.h
Flow_::alstate
void * alstate
Definition: flow.h:472
util-buffer.h
OutputModule_::TxLogFunc
TxLogger TxLogFunc
Definition: output.h:70
OutputModule_::tc_log_progress
int tc_log_progress
Definition: output.h:79
util-lua-tls.h
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
output-lua.h
PacketAlert_
Definition: decode.h:263
STREAMING_TCP_DATA
@ STREAMING_TCP_DATA
Definition: output-streaming.h:37
util-lua-http.h
OutputModule_
Definition: output.h:56
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:446
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
StatsTable_::nstats
uint32_t nstats
Definition: output-stats.h:42
PKT_IS_IPV4
#define PKT_IS_IPV4(p)
Definition: decode.h:245
SCMutex
#define SCMutex
Definition: threads-debug.h:114
app-layer-ssl.h
CreateTimeString
void CreateTimeString(const SCTime_t ts, char *str, size_t size)
Definition: util-time.c:272
output.h
app-layer.h
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:814