suricata
output-lua.c
Go to the documentation of this file.
1 /* Copyright (C) 2014-2022 Open Information Security Foundation
2  *
3  * You can copy, redistribute or modify this Program under the terms of
4  * the GNU General Public License version 2 as published by the Free
5  * Software Foundation.
6  *
7  * This program is distributed in the hope that it will be useful,
8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  * GNU General Public License for more details.
11  *
12  * You should have received a copy of the GNU General Public License
13  * version 2 along with this program; if not, write to the Free Software
14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15  * 02110-1301, USA.
16  */
17 
18 /**
19  * \file
20  *
21  * \author Victor Julien <victor@inliniac.net>
22  *
23  */
24 
25 #include "suricata-common.h"
26 #include "output-lua.h"
27 
28 #ifdef HAVE_LUA
29 #include "util-print.h"
30 #include "util-unittest.h"
31 #include "util-debug.h"
32 #include "output.h"
33 #include "app-layer-htp.h"
34 #include "app-layer.h"
35 #include "app-layer-ssl.h"
36 #include "app-layer-ssh.h"
37 #include "app-layer-parser.h"
38 #include "util-privs.h"
39 #include "util-buffer.h"
40 #include "util-proto-name.h"
41 #include "util-logopenfile.h"
42 #include "util-time.h"
43 #include "util-lua.h"
44 #include "util-lua-common.h"
45 #include "util-lua-http.h"
46 #include "util-lua-dns.h"
47 #include "util-lua-ja3.h"
48 #include "util-lua-tls.h"
49 #include "util-lua-ssh.h"
50 #include "util-lua-hassh.h"
51 #include "util-lua-smtp.h"
52 
53 #define MODULE_NAME "LuaLog"
54 
55 /** \brief structure containing global config
56  * The OutputLuaLogInitSub which is run per script
57  * can access this to get global config info through
58  * it's parent_ctx->data ptr.
59  */
60 typedef struct LogLuaMasterCtx_ {
61  char path[PATH_MAX]; /**< contains script-dir */
62 } LogLuaMasterCtx;
63 
64 typedef struct LogLuaCtx_ {
65  SCMutex m;
66  lua_State *luastate;
67  int deinit_once;
68 } LogLuaCtx;
69 
70 typedef struct LogLuaThreadCtx_ {
71  LogLuaCtx *lua_ctx;
72 } LogLuaThreadCtx;
73 
74 static TmEcode LuaLogThreadInit(ThreadVars *t, const void *initdata, void **data);
75 static TmEcode LuaLogThreadDeinit(ThreadVars *t, void *data);
76 
77 /** \internal
78  * \brief TX logger for lua scripts
79  *
80  * A single call to this function will run one script on a single
81  * transaction.
82  *
83  * NOTE: The flow (f) also referenced by p->flow is locked.
84  */
85 static int LuaTxLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *alstate, void *txptr, uint64_t tx_id)
86 {
87  SCEnter();
88 
89  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
90 
91  SCMutexLock(&td->lua_ctx->m);
92 
93  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
94  LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
95  LuaStateSetTX(td->lua_ctx->luastate, txptr, tx_id);
96  LuaStateSetFlow(td->lua_ctx->luastate, f);
97 
98  /* prepare data to pass to script */
99  lua_getglobal(td->lua_ctx->luastate, "log");
100  lua_newtable(td->lua_ctx->luastate);
101  LuaPushTableKeyValueInt(td->lua_ctx->luastate, "tx_id", (int)(tx_id));
102 
103  int retval = lua_pcall(td->lua_ctx->luastate, 1, 0, 0);
104  if (retval != 0) {
105  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
106  }
107 
108  SCMutexUnlock(&td->lua_ctx->m);
109  SCReturnInt(0);
110 }
111 
112 /** \internal
113  * \brief Streaming logger for lua scripts
114  *
115  * Hooks into the Streaming Logger API. Gets called for each chunk of new
116  * streaming data.
117  */
118 static int LuaStreamingLogger(ThreadVars *tv, void *thread_data, const Flow *f,
119  const uint8_t *data, uint32_t data_len, uint64_t tx_id, uint8_t flags)
120 {
121  SCEnter();
122 
123  void *txptr = NULL;
124  LuaStreamingBuffer b = { data, data_len, flags };
125 
126  SCLogDebug("flags %02x", flags);
127 
129  if (f && f->alstate)
130  txptr = AppLayerParserGetTx(f->proto, f->alproto, f->alstate, tx_id);
131  }
132 
133  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
134 
135  SCMutexLock(&td->lua_ctx->m);
136 
137  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
139  LuaStateSetTX(td->lua_ctx->luastate, txptr, tx_id);
140  LuaStateSetFlow(td->lua_ctx->luastate, (Flow *)f);
141  LuaStateSetStreamingBuffer(td->lua_ctx->luastate, &b);
142 
143  /* prepare data to pass to script */
144  lua_getglobal(td->lua_ctx->luastate, "log");
145  lua_newtable(td->lua_ctx->luastate);
146 
148  LuaPushTableKeyValueInt(td->lua_ctx->luastate, "tx_id", (int)(tx_id));
149 
150  int retval = lua_pcall(td->lua_ctx->luastate, 1, 0, 0);
151  if (retval != 0) {
152  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
153  }
154 
155  SCMutexUnlock(&td->lua_ctx->m);
156 
158 }
159 
160 /** \internal
161  * \brief Packet Logger for lua scripts, for alerts
162  *
163  * A single call to this function will run one script for a single
164  * packet. If it is called, it means that the registered condition
165  * function has returned true.
166  *
167  * The script is called once for each alert stored in the packet.
168  *
169  * NOTE: p->flow is UNlocked
170  */
171 static int LuaPacketLoggerAlerts(ThreadVars *tv, void *thread_data, const Packet *p)
172 {
173  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
174 
175  char timebuf[64];
176  CreateTimeString(p->ts, timebuf, sizeof(timebuf));
177 
178  if (!(PacketIsIPv4(p)) && !(PacketIsIPv6(p))) {
179  /* decoder event */
180  goto not_supported;
181  }
182 
183  /* loop through alerts stored in the packet */
184  SCMutexLock(&td->lua_ctx->m);
185  uint16_t cnt;
186  for (cnt = 0; cnt < p->alerts.cnt; cnt++) {
187  const PacketAlert *pa = &p->alerts.alerts[cnt];
188  if (unlikely(pa->s == NULL)) {
189  continue;
190  }
191 
192  lua_getglobal(td->lua_ctx->luastate, "log");
193 
194  void *txptr = NULL;
195  if (p->flow && p->flow->alstate && (pa->flags & PACKET_ALERT_FLAG_TX))
196  txptr = AppLayerParserGetTx(
197  p->flow->proto, p->flow->alproto, p->flow->alstate, pa->tx_id);
198 
199  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
200  LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
201  LuaStateSetTX(td->lua_ctx->luastate, txptr, pa->tx_id);
202  LuaStateSetFlow(td->lua_ctx->luastate, p->flow);
203  LuaStateSetPacketAlert(td->lua_ctx->luastate, (PacketAlert *)pa);
204 
205  /* prepare data to pass to script */
206  //lua_newtable(td->lua_ctx->luastate);
207 
208  int retval = lua_pcall(td->lua_ctx->luastate, 0, 0, 0);
209  if (retval != 0) {
210  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
211  }
212  }
213  SCMutexUnlock(&td->lua_ctx->m);
214 not_supported:
215  SCReturnInt(0);
216 }
217 
218 static bool LuaPacketConditionAlerts(ThreadVars *tv, void *data, const Packet *p)
219 {
220  return (p->alerts.cnt > 0);
221 }
222 
223 /** \internal
224  * \brief Packet Logger for lua scripts, for packets
225  *
226  * A single call to this function will run one script for a single
227  * packet. If it is called, it means that the registered condition
228  * function has returned true.
229  *
230  * The script is called once for each packet.
231  *
232  * NOTE: p->flow is UNlocked
233  */
234 static int LuaPacketLogger(ThreadVars *tv, void *thread_data, const Packet *p)
235 {
236  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
237 
238  char timebuf[64];
239 
240  if ((!(PacketIsIPv4(p))) && (!(PacketIsIPv6(p)))) {
241  goto not_supported;
242  }
243 
244  CreateTimeString(p->ts, timebuf, sizeof(timebuf));
245 
246  /* loop through alerts stored in the packet */
247  SCMutexLock(&td->lua_ctx->m);
248  lua_getglobal(td->lua_ctx->luastate, "log");
249 
250  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
251  LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
252  LuaStateSetFlow(td->lua_ctx->luastate, p->flow);
253 
254  /* prepare data to pass to script */
255  lua_newtable(td->lua_ctx->luastate);
256 
257  int retval = lua_pcall(td->lua_ctx->luastate, 1, 0, 0);
258  if (retval != 0) {
259  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
260  }
261  SCMutexUnlock(&td->lua_ctx->m);
262 not_supported:
263  SCReturnInt(0);
264 }
265 
266 static bool LuaPacketCondition(ThreadVars *tv, void *data, const Packet *p)
267 {
268  return true;
269 }
270 
271 /** \internal
272  * \brief File API Logger function for Lua scripts
273  *
274  * Executes a script once for one file.
275  *
276  * NOTE p->flow is locked at this point
277  */
278 static int LuaFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff,
279  void *tx, const uint64_t tx_id, uint8_t dir)
280 {
281  SCEnter();
282  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
283 
284  if ((!(PacketIsIPv4(p))) && (!(PacketIsIPv6(p))))
285  return 0;
286 
287  BUG_ON(ff->flags & FILE_LOGGED);
288 
289  SCLogDebug("ff %p", ff);
290 
291  SCMutexLock(&td->lua_ctx->m);
292 
293  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
294  LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
295  LuaStateSetTX(td->lua_ctx->luastate, tx, tx_id);
296  LuaStateSetFlow(td->lua_ctx->luastate, p->flow);
297  LuaStateSetFile(td->lua_ctx->luastate, (File *)ff);
298 
299  /* get the lua function to call */
300  lua_getglobal(td->lua_ctx->luastate, "log");
301 
302  int retval = lua_pcall(td->lua_ctx->luastate, 0, 0, 0);
303  if (retval != 0) {
304  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
305  }
306  SCMutexUnlock(&td->lua_ctx->m);
307  return 0;
308 }
309 
310 /** \internal
311  * \brief Flow API Logger function for Lua scripts
312  *
313  * Executes a script once for one flow
314  *
315  * Note: flow 'f' is locked
316  */
317 static int LuaFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
318 {
319  SCEnter();
320  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
321 
322  SCLogDebug("f %p", f);
323 
324  SCMutexLock(&td->lua_ctx->m);
325 
326  LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
327  LuaStateSetFlow(td->lua_ctx->luastate, f);
328 
329  /* get the lua function to call */
330  lua_getglobal(td->lua_ctx->luastate, "log");
331 
332  int retval = lua_pcall(td->lua_ctx->luastate, 0, 0, 0);
333  if (retval != 0) {
334  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
335  }
336  SCMutexUnlock(&td->lua_ctx->m);
337  return 0;
338 }
339 
340 
341 
342 static int LuaStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable *st)
343 {
344  SCEnter();
345  LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
346 
347  SCMutexLock(&td->lua_ctx->m);
348 
349  lua_State *luastate = td->lua_ctx->luastate;
350  /* get the lua function to call */
351  lua_getglobal(td->lua_ctx->luastate, "log");
352 
353  /* create lua array, which is really just a table. The key is an int (1-x),
354  * the value another table with named fields: name, tm_name, value, pvalue.
355  * { 1, { name=<name>, tmname=<tm_name>, value=<value>, pvalue=<pvalue>}}
356  * { 2, { name=<name>, tmname=<tm_name>, value=<value>, pvalue=<pvalue>}}
357  * etc
358  */
359  lua_newtable(luastate);
360  uint32_t u = 0;
361  for (; u < st->nstats; u++) {
362  lua_pushinteger(luastate, u + 1);
363 
364  lua_newtable(luastate);
365 
366  lua_pushstring(luastate, "name");
367  lua_pushstring(luastate, st->stats[u].name);
368  lua_settable(luastate, -3);
369 
370  lua_pushstring(luastate, "tmname");
371  lua_pushstring(luastate, st->stats[u].tm_name);
372  lua_settable(luastate, -3);
373 
374  lua_pushstring(luastate, "value");
375  lua_pushinteger(luastate, st->stats[u].value);
376  lua_settable(luastate, -3);
377 
378  lua_pushstring(luastate, "pvalue");
379  lua_pushinteger(luastate, st->stats[u].pvalue);
380  lua_settable(luastate, -3);
381 
382  lua_settable(luastate, -3);
383  }
384 
385  int retval = lua_pcall(td->lua_ctx->luastate, 1, 0, 0);
386  if (retval != 0) {
387  SCLogInfo("failed to run script: %s", lua_tostring(td->lua_ctx->luastate, -1));
388  }
389  SCMutexUnlock(&td->lua_ctx->m);
390  return 0;
391 
392 }
393 
394 typedef struct LogLuaScriptOptions_ {
395  AppProto alproto;
396  int packet;
397  int alerts;
398  int file;
399  int streaming;
400  int tcp_data;
401  int http_body;
402  int flow;
403  int stats;
404 } LogLuaScriptOptions;
405 
406 /** \brief load and evaluate the script
407  *
408  * This function parses the script, checks if all the required functions
409  * are defined and runs the 'init' function. The init function will inform
410  * us what the scripts needs are.
411  *
412  * \param filename filename of lua script file
413  * \param options struct to pass script requirements/options back to caller
414  * \retval errcode 0 ok, -1 error
415  */
416 static int LuaScriptInit(const char *filename, LogLuaScriptOptions *options) {
417  lua_State *luastate = LuaGetState();
418  if (luastate == NULL)
419  goto error;
420  luaL_openlibs(luastate);
421 
422  int status = luaL_loadfile(luastate, filename);
423  if (status) {
424  SCLogError("couldn't load file: %s", lua_tostring(luastate, -1));
425  goto error;
426  }
427 
428  /* prime the script (or something) */
429  if (lua_pcall(luastate, 0, 0, 0) != 0) {
430  SCLogError("couldn't prime file: %s", lua_tostring(luastate, -1));
431  goto error;
432  }
433 
434  lua_getglobal(luastate, "init");
435  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
436  SCLogError("no init function in script");
437  goto error;
438  }
439 
440  lua_newtable(luastate); /* stack at -1 */
441  if (lua_gettop(luastate) == 0 || lua_type(luastate, 2) != LUA_TTABLE) {
442  SCLogError("no table setup");
443  goto error;
444  }
445 
446  lua_pushliteral(luastate, "script_api_ver");
447  lua_pushnumber (luastate, 1);
448  lua_settable(luastate, -3);
449 
450  if (lua_pcall(luastate, 1, 1, 0) != 0) {
451  SCLogError("couldn't run script 'init' function: %s", lua_tostring(luastate, -1));
452  goto error;
453  }
454 
455  /* process returns from script */
456  if (lua_gettop(luastate) == 0) {
457  SCLogError("init function in script should return table, nothing returned");
458  goto error;
459  }
460  if (lua_type(luastate, 1) != LUA_TTABLE) {
461  SCLogError("init function in script should return table, returned is not table");
462  goto error;
463  }
464 
465  lua_pushnil(luastate);
466  const char *k, *v;
467  while (lua_next(luastate, -2)) {
468  k = lua_tostring(luastate, -2);
469  if (k == NULL)
470  continue;
471 
472  v = lua_tostring(luastate, -1);
473  lua_pop(luastate, 1);
474  if (v == NULL)
475  continue;
476 
477  SCLogDebug("k='%s', v='%s'", k, v);
478 
479  if (strcmp(k,"protocol") == 0 && strcmp(v, "http") == 0)
480  options->alproto = ALPROTO_HTTP1;
481  else if (strcmp(k,"protocol") == 0 && strcmp(v, "dns") == 0)
482  options->alproto = ALPROTO_DNS;
483  else if (strcmp(k,"protocol") == 0 && strcmp(v, "tls") == 0)
484  options->alproto = ALPROTO_TLS;
485  else if (strcmp(k,"protocol") == 0 && strcmp(v, "ssh") == 0)
486  options->alproto = ALPROTO_SSH;
487  else if (strcmp(k,"protocol") == 0 && strcmp(v, "smtp") == 0)
488  options->alproto = ALPROTO_SMTP;
489  else if (strcmp(k, "type") == 0 && strcmp(v, "packet") == 0)
490  options->packet = 1;
491  else if (strcmp(k, "filter") == 0 && strcmp(v, "alerts") == 0)
492  options->alerts = 1;
493  else if (strcmp(k, "type") == 0 && strcmp(v, "file") == 0)
494  options->file = 1;
495  else if (strcmp(k, "type") == 0 && strcmp(v, "streaming") == 0)
496  options->streaming = 1;
497  else if (strcmp(k, "type") == 0 && strcmp(v, "flow") == 0)
498  options->flow = 1;
499  else if (strcmp(k, "filter") == 0 && strcmp(v, "tcp") == 0)
500  options->tcp_data = 1;
501  else if (strcmp(k, "type") == 0 && strcmp(v, "stats") == 0)
502  options->stats = 1;
503  else
504  SCLogInfo("unknown key and/or value: k='%s', v='%s'", k, v);
505  }
506 
507  if (((options->alproto != ALPROTO_UNKNOWN)) + options->packet + options->file > 1) {
508  SCLogError("invalid combination of 'needs' in the script");
509  goto error;
510  }
511 
512  lua_getglobal(luastate, "setup");
513  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
514  SCLogError("no setup function in script");
515  goto error;
516  }
517 
518  lua_getglobal(luastate, "log");
519  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
520  SCLogError("no log function in script");
521  goto error;
522  }
523 
524  lua_getglobal(luastate, "deinit");
525  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
526  SCLogError("no deinit function in script");
527  goto error;
528  }
529 
530  LuaReturnState(luastate);
531  return 0;
532 error:
533  if (luastate)
534  LuaReturnState(luastate);
535  return -1;
536 }
537 
538 /** \brief setup a luastate for use at runtime
539  *
540  * This loads the script, primes it and then runs the 'setup' function.
541  *
542  * \retval state Returns the set up luastate on success, NULL on error
543  */
544 static lua_State *LuaScriptSetup(const char *filename)
545 {
546  lua_State *luastate = LuaGetState();
547  if (luastate == NULL) {
548  SCLogError("luaL_newstate failed");
549  goto error;
550  }
551 
552  luaL_openlibs(luastate);
553 
554  int status = luaL_loadfile(luastate, filename);
555  if (status) {
556  SCLogError("couldn't load file: %s", lua_tostring(luastate, -1));
557  goto error;
558  }
559 
560  /* prime the script */
561  if (lua_pcall(luastate, 0, 0, 0) != 0) {
562  SCLogError("couldn't prime file: %s", lua_tostring(luastate, -1));
563  goto error;
564  }
565 
566  lua_getglobal(luastate, "setup");
567 
568  /* register functions common to all */
569  LuaRegisterFunctions(luastate);
570  /* unconditionally register http function. They will only work
571  * if the tx is registered in the state at runtime though. */
572  LuaRegisterHttpFunctions(luastate);
573  LuaRegisterDnsFunctions(luastate);
574  LuaRegisterJa3Functions(luastate);
575  LuaRegisterTlsFunctions(luastate);
576  LuaRegisterSshFunctions(luastate);
577  LuaRegisterHasshFunctions(luastate);
578  LuaRegisterSmtpFunctions(luastate);
579 
580  if (lua_pcall(luastate, 0, 0, 0) != 0) {
581  SCLogError("couldn't run script 'setup' function: %s", lua_tostring(luastate, -1));
582  goto error;
583  }
584 
585  SCLogDebug("lua_State %p is set up", luastate);
586  return luastate;
587 error:
588  if (luastate)
589  LuaReturnState(luastate);
590  return NULL;
591 }
592 
593 static void LogLuaSubFree(OutputCtx *oc) {
594  if (oc->data)
595  SCFree(oc->data);
596  SCFree(oc);
597 }
598 
599 /** \brief initialize output for a script instance
600  *
601  * Runs script 'setup' function.
602  */
603 static OutputInitResult OutputLuaLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
604 {
605  OutputInitResult result = { NULL, false };
606  if (conf == NULL)
607  return result;
608 
609  LogLuaCtx *lua_ctx = SCCalloc(1, sizeof(LogLuaCtx));
610  if (unlikely(lua_ctx == NULL))
611  return result;
612 
613  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
614  if (unlikely(output_ctx == NULL)) {
615  SCFree(lua_ctx);
616  return result;
617  }
618 
619  SCMutexInit(&lua_ctx->m, NULL);
620 
621  const char *dir = "";
622  if (parent_ctx && parent_ctx->data) {
623  LogLuaMasterCtx *mc = parent_ctx->data;
624  dir = mc->path;
625  }
626 
627  char path[PATH_MAX] = "";
628  int ret = snprintf(path, sizeof(path),"%s%s%s", dir, strlen(dir) ? "/" : "", conf->val);
629  if (ret < 0 || ret == sizeof(path)) {
630  SCLogError("failed to construct lua script path");
631  goto error;
632  }
633  SCLogDebug("script full path %s", path);
634 
635  SCMutexLock(&lua_ctx->m);
636  lua_ctx->luastate = LuaScriptSetup(path);
637  SCMutexUnlock(&lua_ctx->m);
638  if (lua_ctx->luastate == NULL)
639  goto error;
640 
641  SCLogDebug("lua_ctx %p", lua_ctx);
642 
643  output_ctx->data = lua_ctx;
644  output_ctx->DeInit = LogLuaSubFree;
645 
646  result.ctx = output_ctx;
647  result.ok = true;
648  return result;
649 error:
650  SCMutexDestroy(&lua_ctx->m);
651  SCFree(lua_ctx);
652  SCFree(output_ctx);
653  return result;
654 }
655 
656 static void LogLuaMasterFree(OutputCtx *oc)
657 {
658  if (oc->data)
659  SCFree(oc->data);
660 
661  OutputModule *om, *tom;
662  TAILQ_FOREACH_SAFE(om, &oc->submodules, entries, tom) {
663  SCFree(om);
664  }
665  SCFree(oc);
666 }
667 
668 /** \internal
669  * \brief initialize output instance for lua module
670  *
671  * Parses nested script list, primes them to find out what they
672  * inspect, then fills the OutputCtx::submodules list with the
673  * proper Logger function for the data type the script needs.
674  */
675 static OutputInitResult OutputLuaLogInit(ConfNode *conf)
676 {
677  OutputInitResult result = { NULL, false };
678  const char *dir = ConfNodeLookupChildValue(conf, "scripts-dir");
679  if (dir == NULL)
680  dir = "";
681 
682  ConfNode *scripts = ConfNodeLookupChild(conf, "scripts");
683  if (scripts == NULL) {
684  /* No "outputs" section in the configuration. */
685  SCLogInfo("scripts not defined");
686  return result;
687  }
688 
689  /* global output ctx setup */
690  OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
691  if (unlikely(output_ctx == NULL)) {
692  return result;
693  }
694  output_ctx->DeInit = LogLuaMasterFree;
695  output_ctx->data = SCCalloc(1, sizeof(LogLuaMasterCtx));
696  if (unlikely(output_ctx->data == NULL)) {
697  SCFree(output_ctx);
698  return result;
699  }
700  LogLuaMasterCtx *master_config = output_ctx->data;
701  strlcpy(master_config->path, dir, sizeof(master_config->path));
702  TAILQ_INIT(&output_ctx->submodules);
703 
704  /* check the enables scripts and set them up as submodules */
705  ConfNode *script;
706  TAILQ_FOREACH(script, &scripts->head, next) {
707  SCLogInfo("enabling script %s", script->val);
708  LogLuaScriptOptions opts;
709  memset(&opts, 0x00, sizeof(opts));
710 
711  char path[PATH_MAX] = "";
712  snprintf(path, sizeof(path),"%s%s%s", dir, strlen(dir) ? "/" : "", script->val);
713  SCLogDebug("script full path %s", path);
714 
715  int r = LuaScriptInit(path, &opts);
716  if (r != 0) {
717  SCLogError("couldn't initialize script");
718  goto error;
719  }
720 
721  /* create an OutputModule for this script, based
722  * on it's needs. */
723  OutputModule *om = SCCalloc(1, sizeof(*om));
724  if (om == NULL) {
725  SCLogError("calloc() failed");
726  goto error;
727  }
728 
729  om->name = MODULE_NAME;
730  om->conf_name = script->val;
731  om->InitSubFunc = OutputLuaLogInitSub;
732  om->ThreadInit = LuaLogThreadInit;
733  om->ThreadDeinit = LuaLogThreadDeinit;
734 
735  if (opts.alproto == ALPROTO_HTTP1 && opts.streaming) {
736  om->StreamingLogFunc = LuaStreamingLogger;
738  om->alproto = ALPROTO_HTTP1;
741  } else if (opts.alproto == ALPROTO_HTTP1) {
742  om->TxLogFunc = LuaTxLogger;
743  om->alproto = ALPROTO_HTTP1;
744  om->ts_log_progress = -1;
745  om->tc_log_progress = -1;
747  } else if (opts.alproto == ALPROTO_TLS) {
748  om->TxLogFunc = LuaTxLogger;
749  om->alproto = ALPROTO_TLS;
753  } else if (opts.alproto == ALPROTO_DNS) {
754  om->TxLogFunc = LuaTxLogger;
755  om->alproto = ALPROTO_DNS;
756  om->ts_log_progress = -1;
757  om->tc_log_progress = -1;
760  } else if (opts.alproto == ALPROTO_SSH) {
761  om->TxLogFunc = LuaTxLogger;
762  om->alproto = ALPROTO_SSH;
765  } else if (opts.alproto == ALPROTO_SMTP) {
766  om->TxLogFunc = LuaTxLogger;
767  om->alproto = ALPROTO_SMTP;
768  om->ts_log_progress = -1;
769  om->tc_log_progress = -1;
771  } else if (opts.packet && opts.alerts) {
772  om->PacketLogFunc = LuaPacketLoggerAlerts;
773  om->PacketConditionFunc = LuaPacketConditionAlerts;
774  } else if (opts.packet && opts.alerts == 0) {
775  om->PacketLogFunc = LuaPacketLogger;
776  om->PacketConditionFunc = LuaPacketCondition;
777  } else if (opts.file) {
778  om->FileLogFunc = LuaFileLogger;
780  } else if (opts.streaming && opts.tcp_data) {
781  om->StreamingLogFunc = LuaStreamingLogger;
783  } else if (opts.flow) {
784  om->FlowLogFunc = LuaFlowLogger;
785  } else if (opts.stats) {
786  om->StatsLogFunc = LuaStatsLogger;
787  } else {
788  SCLogError("failed to setup thread module");
789  SCFree(om);
790  goto error;
791  }
792 
793  TAILQ_INSERT_TAIL(&output_ctx->submodules, om, entries);
794  }
795 
796  result.ctx = output_ctx;
797  result.ok = true;
798  return result;
799 
800 error:
801  if (output_ctx->DeInit)
802  output_ctx->DeInit(output_ctx);
803 
804  int failure_fatal = 0;
805  if (ConfGetBool("engine.init-failure-fatal", &failure_fatal) != 1) {
806  SCLogDebug("ConfGetBool could not load the value.");
807  }
808  if (failure_fatal) {
809  FatalError("Error during setup of lua output. Details should be "
810  "described in previous error messages. Shutting down...");
811  }
812 
813  return result;
814 }
815 
816 /** \internal
817  * \brief Run the scripts 'deinit' function
818  */
819 static void OutputLuaLogDoDeinit(LogLuaCtx *lua_ctx)
820 {
821  lua_State *luastate = lua_ctx->luastate;
822 
823  lua_getglobal(luastate, "deinit");
824  if (lua_type(luastate, -1) != LUA_TFUNCTION) {
825  SCLogError("no deinit function in script");
826  return;
827  }
828  //LuaPrintStack(luastate);
829 
830  if (lua_pcall(luastate, 0, 0, 0) != 0) {
831  SCLogError("couldn't run script 'deinit' function: %s", lua_tostring(luastate, -1));
832  return;
833  }
834  LuaReturnState(luastate);
835 }
836 
837 /** \internal
838  * \brief Initialize the thread storage for lua
839  *
840  * Currently only stores a pointer to the global LogLuaCtx
841  */
842 static TmEcode LuaLogThreadInit(ThreadVars *t, const void *initdata, void **data)
843 {
844  LogLuaThreadCtx *td = SCCalloc(1, sizeof(*td));
845  if (unlikely(td == NULL))
846  return TM_ECODE_FAILED;
847 
848  if (initdata == NULL) {
849  SCLogDebug("Error getting context for LuaLog. \"initdata\" argument NULL");
850  SCFree(td);
851  return TM_ECODE_FAILED;
852  }
853 
854  LogLuaCtx *lua_ctx = ((OutputCtx *)initdata)->data;
855  SCLogDebug("lua_ctx %p", lua_ctx);
856  td->lua_ctx = lua_ctx;
857  *data = (void *)td;
858  return TM_ECODE_OK;
859 }
860 
861 /** \internal
862  * \brief Deinit the thread storage for lua
863  *
864  * Calls OutputLuaLogDoDeinit if no-one else already did.
865  */
866 static TmEcode LuaLogThreadDeinit(ThreadVars *t, void *data)
867 {
868  LogLuaThreadCtx *td = (LogLuaThreadCtx *)data;
869  if (td == NULL) {
870  return TM_ECODE_OK;
871  }
872 
873  SCMutexLock(&td->lua_ctx->m);
874  if (td->lua_ctx->deinit_once == 0) {
875  OutputLuaLogDoDeinit(td->lua_ctx);
876  td->lua_ctx->deinit_once = 1;
877  }
878  SCMutexUnlock(&td->lua_ctx->m);
879 
880  /* clear memory */
881  memset(td, 0, sizeof(*td));
882 
883  SCFree(td);
884  return TM_ECODE_OK;
885 }
886 
887 void LuaLogRegister(void) {
888  /* register as separate module */
889  OutputRegisterModule(MODULE_NAME, "lua", OutputLuaLogInit);
890 }
891 
892 #else /* HAVE_LUA */
893 
894 void LuaLogRegister (void) {
895  /* no-op */
896 }
897 
898 #endif /* HAVE_LUA */
AppLayerHtpNeedFileInspection
void AppLayerHtpNeedFileInspection(void)
Sets a flag that informs the HTP app layer that some module in the engine needs the http request file...
Definition: app-layer-htp.c:516
util-lua-ssh.h
PacketAlert_::s
const struct Signature_ * s
Definition: decode.h:252
app-layer-ssh.h
util-lua-hassh.h
util-lua-common.h
PACKET_ALERT_FLAG_TX
#define PACKET_ALERT_FLAG_TX
Definition: decode.h:264
TAILQ_INIT
#define TAILQ_INIT(head)
Definition: queue.h:262
ALPROTO_DNS
@ ALPROTO_DNS
Definition: app-layer-protos.h:41
ConfNode_::val
char * val
Definition: conf.h:34
ConfGetBool
int ConfGetBool(const char *name, int *val)
Retrieve a configuration value as a boolean.
Definition: conf.c:483
unlikely
#define unlikely(expr)
Definition: util-optimize.h:35
ALPROTO_TLS
@ ALPROTO_TLS
Definition: app-layer-protos.h:33
SCLogDebug
#define SCLogDebug(...)
Definition: util-debug.h:269
next
struct HtpBodyChunk_ * next
Definition: app-layer-htp.h:0
Flow_::proto
uint8_t proto
Definition: flow.h:378
util-lua.h
AppProto
uint16_t AppProto
Definition: app-layer-protos.h:82
PacketAlerts_::cnt
uint16_t cnt
Definition: decode.h:274
OutputModule_::name
const char * name
Definition: output.h:58
OutputModule_::StreamingLogFunc
StreamingLogger StreamingLogFunc
Definition: output.h:75
Flow_
Flow data structure.
Definition: flow.h:356
OutputModule_::FileLogFunc
FileLogger FileLogFunc
Definition: output.h:72
util-lua-ja3.h
OutputModule_::ts_log_progress
int ts_log_progress
Definition: output.h:80
TAILQ_FOREACH
#define TAILQ_FOREACH(var, head, field)
Definition: queue.h:252
PacketAlerts_::alerts
PacketAlert * alerts
Definition: decode.h:277
SSHTxLogCondition
bool SSHTxLogCondition(ThreadVars *tv, const Packet *p, void *state, void *tx, uint64_t tx_id)
Definition: app-layer-ssh.c:74
OutputModule_::StatsLogFunc
StatsLogger StatsLogFunc
Definition: output.h:76
SCMutexLock
#define SCMutexLock(mut)
Definition: threads-debug.h:117
util-privs.h
TAILQ_INSERT_TAIL
#define TAILQ_INSERT_TAIL(head, elm, field)
Definition: queue.h:294
m
SCMutex m
Definition: flow-hash.h:6
OutputModule_::InitSubFunc
OutputInitSubFunc InitSubFunc
Definition: output.h:62
ALPROTO_SSH
@ ALPROTO_SSH
Definition: app-layer-protos.h:34
StatsRecord_::pvalue
int64_t pvalue
Definition: output-stats.h:36
OutputModule_::PacketLogFunc
PacketLogger PacketLogFunc
Definition: output.h:68
TM_ECODE_FAILED
@ TM_ECODE_FAILED
Definition: tm-threads-common.h:85
OUTPUT_STREAMING_FLAG_TRANSACTION
#define OUTPUT_STREAMING_FLAG_TRANSACTION
Definition: output-streaming.h:33
OutputModule_::alproto
AppProto alproto
Definition: output.h:77
Packet_::alerts
PacketAlerts alerts
Definition: decode.h:603
util-unittest.h
OutputCtx_::data
void * data
Definition: tm-modules.h:88
TM_ECODE_OK
@ TM_ECODE_OK
Definition: tm-threads-common.h:84
PacketAlert_::tx_id
uint64_t tx_id
Definition: decode.h:253
OutputCtx_
Definition: tm-modules.h:85
OutputModule_::stream_type
enum OutputStreamingType stream_type
Definition: output.h:78
strlcpy
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: util-strlcpyu.c:43
MODULE_NAME
#define MODULE_NAME
Definition: alert-debuglog.c:61
app-layer-htp.h
OutputModule_::ThreadInit
ThreadInitFunc ThreadInit
Definition: output.h:64
util-debug.h
OutputInitResult_::ctx
OutputCtx * ctx
Definition: output.h:47
OutputModule_::ThreadDeinit
ThreadDeinitFunc ThreadDeinit
Definition: output.h:65
Packet_::ts
SCTime_t ts
Definition: decode.h:530
SCMutexUnlock
#define SCMutexUnlock(mut)
Definition: threads-debug.h:119
ALPROTO_SMTP
@ ALPROTO_SMTP
Definition: app-layer-protos.h:32
TLS_HANDSHAKE_DONE
@ TLS_HANDSHAKE_DONE
Definition: app-layer-ssl.h:79
AppLayerHtpEnableRequestBodyCallback
void AppLayerHtpEnableRequestBodyCallback(void)
Sets a flag that informs the HTP app layer that some module in the engine needs the http request body...
Definition: app-layer-htp.c:474
AppLayerParserRegisterLogger
void AppLayerParserRegisterLogger(uint8_t ipproto, AppProto alproto)
Definition: app-layer-parser.c:463
OutputRegisterModule
void OutputRegisterModule(const char *, const char *, OutputInitFunc)
util-print.h
SCEnter
#define SCEnter(...)
Definition: util-debug.h:271
StatsRecord_::name
const char * name
Definition: output-stats.h:32
ThreadVars_
Per thread variable structure.
Definition: threadvars.h:57
util-time.h
OutputInitResult_::ok
bool ok
Definition: output.h:48
app-layer-parser.h
BUG_ON
#define BUG_ON(x)
Definition: suricata-common.h:300
StatsTable_
Definition: output-stats.h:39
StatsRecord_::value
int64_t value
Definition: output-stats.h:35
OutputModule_::conf_name
const char * conf_name
Definition: output.h:59
OutputModule_::FlowLogFunc
FlowLogger FlowLogFunc
Definition: output.h:74
Packet_
Definition: decode.h:482
TmEcode
TmEcode
Definition: tm-threads-common.h:83
AppLayerHtpEnableResponseBodyCallback
void AppLayerHtpEnableResponseBodyCallback(void)
Sets a flag that informs the HTP app layer that some module in the engine needs the http request body...
Definition: app-layer-htp.c:487
util-proto-name.h
SCLogInfo
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
Definition: util-debug.h:224
TAILQ_FOREACH_SAFE
#define TAILQ_FOREACH_SAFE(var, head, field, tvar)
Definition: queue.h:329
AppLayerParserGetTx
void * AppLayerParserGetTx(uint8_t ipproto, AppProto alproto, void *alstate, uint64_t tx_id)
Definition: app-layer-parser.c:1108
SCMutexInit
#define SCMutexInit(mut, mutattrs)
Definition: threads-debug.h:116
StatsTable_::stats
StatsRecord * stats
Definition: output-stats.h:40
util-lua-smtp.h
ConfNodeLookupChild
ConfNode * ConfNodeLookupChild(const ConfNode *node, const char *name)
Lookup a child configuration node by name.
Definition: conf.c:786
PacketAlert_::flags
uint8_t flags
Definition: decode.h:251
File_::flags
uint16_t flags
Definition: util-file.h:80
File_
Definition: util-file.h:79
cnt
uint32_t cnt
Definition: tmqh-packetpool.h:7
OutputInitResult_
Definition: output.h:46
Packet_::flow
struct Flow_ * flow
Definition: decode.h:521
OutputModule_::TxLogCondition
TxLoggerCondition TxLogCondition
Definition: output.h:71
flags
uint8_t flags
Definition: decode-gre.h:0
suricata-common.h
OutputCtx_::DeInit
void(* DeInit)(struct OutputCtx_ *)
Definition: tm-modules.h:91
STREAMING_HTTP_BODIES
@ STREAMING_HTTP_BODIES
Definition: output-streaming.h:37
ALPROTO_HTTP1
@ ALPROTO_HTTP1
Definition: app-layer-protos.h:30
StatsRecord_::tm_name
const char * tm_name
Definition: output-stats.h:34
lua_State
void lua_State
Definition: suricata-common.h:501
util-lua-dns.h
FatalError
#define FatalError(...)
Definition: util-debug.h:502
LuaLogRegister
void LuaLogRegister(void)
Definition: output-lua.c:894
tv
ThreadVars * tv
Definition: fuzz_decodepcapfile.c:32
OutputModule_::PacketConditionFunc
PacketLogCondition PacketConditionFunc
Definition: output.h:69
FILE_LOGGED
#define FILE_LOGGED
Definition: util-file.h:53
SCLogError
#define SCLogError(...)
Macro used to log ERROR messages.
Definition: util-debug.h:261
SCFree
#define SCFree(p)
Definition: util-mem.h:61
ConfNode_
Definition: conf.h:32
util-logopenfile.h
Flow_::alstate
void * alstate
Definition: flow.h:481
util-buffer.h
OutputModule_::TxLogFunc
TxLogger TxLogFunc
Definition: output.h:70
OutputModule_::tc_log_progress
int tc_log_progress
Definition: output.h:79
util-lua-tls.h
ALPROTO_UNKNOWN
@ ALPROTO_UNKNOWN
Definition: app-layer-protos.h:29
output-lua.h
PacketAlert_
Definition: decode.h:248
STREAMING_TCP_DATA
@ STREAMING_TCP_DATA
Definition: output-streaming.h:36
util-lua-http.h
OutputModule_
Definition: output.h:56
Flow_::alproto
AppProto alproto
application level protocol
Definition: flow.h:455
SCCalloc
#define SCCalloc(nm, sz)
Definition: util-mem.h:53
SCReturnInt
#define SCReturnInt(x)
Definition: util-debug.h:275
SCMutexDestroy
#define SCMutexDestroy
Definition: threads-debug.h:120
StatsTable_::nstats
uint32_t nstats
Definition: output-stats.h:42
SCMutex
#define SCMutex
Definition: threads-debug.h:114
app-layer-ssl.h
CreateTimeString
void CreateTimeString(const SCTime_t ts, char *str, size_t size)
Definition: util-time.c:272
output.h
app-layer.h
ConfNodeLookupChildValue
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
Definition: conf.c:814