34 #define MODULE_NAME "OutputFilestore" 38 #define SHA256_STRING_LEN (SHA256_LENGTH * 2) 39 #define LEAF_DIR_MAX_LEN 4 40 #define FILESTORE_PREFIX_MAX (PATH_MAX - SHA256_STRING_LEN - LEAF_DIR_MAX_LEN) 44 static const char *default_log_dir =
"filestore";
49 typedef struct OutputFilestoreCtx_ {
50 char prefix[FILESTORE_PREFIX_MAX];
51 char tmpdir[FILESTORE_PREFIX_MAX];
56 typedef struct OutputFilestoreLogThread_ {
57 OutputFilestoreCtx *ctx;
58 uint16_t counter_max_hits;
59 uint16_t fs_error_counter;
60 } OutputFilestoreLogThread;
66 #define WARN_ONCE(err_code, ...) do { \ 67 if (!once_errs[err_code]) { \ 68 once_errs[err_code] = true; \ 69 SCLogWarning(err_code, __VA_ARGS__); \ 73 static uint64_t OutputFilestoreOpenFilesCounter(
void)
78 static uint32_t g_file_store_max_open_files = 0;
80 static void FileSetMaxOpenFiles(uint32_t count)
82 g_file_store_max_open_files = count;
85 static uint32_t FileGetMaxOpenFiles(
void)
87 return g_file_store_max_open_files;
90 static void PrintHexString(
char *
str,
size_t size, uint8_t *buf,
size_t buf_len)
94 for (i = 0, x = 0; x < buf_len; x++) {
95 i += snprintf(&str[i], size - i,
"%02x", buf[x]);
106 static void OutputFilestoreUpdateFileTime(
const char *src_filename,
107 const char *filename)
110 if (stat(src_filename, &sb) != 0) {
111 SCLogDebug(
"Failed to stat %s: %s", filename, strerror(errno));
114 struct utimbuf utimbuf = {
115 .actime = sb.st_atime,
116 .modtime = sb.st_mtime,
118 if (utime(filename, &utimbuf) != 0) {
119 SCLogDebug(
"Failed to update file timestamps: %s: %s", filename,
124 static void OutputFilestoreFinalizeFiles(
ThreadVars *tv,
125 const OutputFilestoreLogThread *oft,
const OutputFilestoreCtx *ctx,
129 char sha256string[(SHA256_LENGTH * 2) + 1];
130 PrintHexString(sha256string,
sizeof(sha256string), ff->sha256,
133 char tmp_filename[PATH_MAX] =
"";
134 snprintf(tmp_filename,
sizeof(tmp_filename),
"%s/file.%u", ctx->tmpdir,
137 char final_filename[PATH_MAX] =
"";
138 snprintf(final_filename,
sizeof(final_filename),
"%s/%c%c/%s",
139 ctx->prefix, sha256string[0], sha256string[1], sha256string);
142 OutputFilestoreUpdateFileTime(tmp_filename, final_filename);
143 if (unlink(tmp_filename) != 0) {
146 "Failed to remove temporary file %s: %s", tmp_filename,
149 }
else if (rename(tmp_filename, final_filename) != 0) {
152 tmp_filename, final_filename, strerror(errno));
153 if (unlink(tmp_filename) != 0) {
161 #ifdef HAVE_LIBJANSSON 163 char js_metadata_filename[PATH_MAX];
164 if (snprintf(js_metadata_filename,
sizeof(js_metadata_filename),
165 "%s.%"PRIuMAX
".%u.json", final_filename,
167 == (
int)
sizeof(js_metadata_filename)) {
169 "Failed to write file info record. Output filename truncated.");
171 json_t *js_fileinfo = JsonBuildFileInfoRecord(p, ff,
true, dir,
173 if (
likely(js_fileinfo != NULL)) {
174 json_dump_file(js_fileinfo, js_metadata_filename, 0);
175 json_decref(js_fileinfo);
182 static int OutputFilestoreLogger(
ThreadVars *tv,
void *thread_data,
183 const Packet *p,
File *ff,
const uint8_t *data, uint32_t data_len,
184 uint8_t
flags, uint8_t dir)
187 OutputFilestoreLogThread *aft = (OutputFilestoreLogThread *)thread_data;
188 OutputFilestoreCtx *ctx = aft->ctx;
189 char filename[PATH_MAX] =
"";
193 if (p->
flow == NULL) {
201 SCLogDebug(
"ff %p, data %p, data_len %u", ff, data, data_len);
203 char base_filename[PATH_MAX] =
"";
204 snprintf(base_filename,
sizeof(base_filename),
"%s/file.%u",
206 snprintf(filename,
sizeof(filename),
"%s", base_filename);
209 file_fd = open(filename, O_CREAT | O_TRUNC |
O_NOFOLLOW | O_WRONLY,
214 "Filestore (v2) failed to create %s: %s", filename,
219 if (
SC_ATOMIC_GET(filestore_open_file_cnt) < FileGetMaxOpenFiles()) {
223 if (FileGetMaxOpenFiles() > 0) {
229 }
else if (data != NULL) {
231 file_fd = open(filename, O_APPEND |
O_NOFOLLOW | O_WRONLY);
235 "Filestore (v2) failed to open file %s: %s",
236 filename, strerror(errno));
245 ssize_t r = write(file_fd, (
const void *)data, (
size_t)data_len);
249 "Filestore (v2) failed to write to %s: %s",
250 filename, strerror(errno));
267 OutputFilestoreFinalizeFiles(tv, aft, ctx, p, ff, dir);
276 OutputFilestoreLogThread *aft =
SCMalloc(
sizeof(OutputFilestoreLogThread));
279 memset(aft, 0,
sizeof(OutputFilestoreLogThread));
281 if (initdata == NULL) {
282 SCLogDebug(
"Error getting context for LogFileStore. \"initdata\" argument NULL");
287 OutputFilestoreCtx *ctx = ((
OutputCtx *)initdata)->data;
290 aft->counter_max_hits =
304 OutputFilestoreLogThread *aft = (OutputFilestoreLogThread *)data;
310 memset(aft, 0,
sizeof(OutputFilestoreLogThread));
316 static void OutputFilestoreLogDeInitCtx(
OutputCtx *output_ctx)
318 OutputFilestoreCtx *ctx = (OutputFilestoreCtx *)output_ctx->
data;
319 if (ctx->xff_cfg != NULL) {
326 static void GetLogDirectory(
const ConfNode *conf,
char *out,
size_t out_size)
329 if (log_base_dir == NULL) {
330 SCLogConfig(
"Filestore (v2) default log directory %s", default_log_dir);
331 log_base_dir = default_log_dir;
334 strlcpy(out, log_base_dir, out_size);
337 snprintf(out, out_size,
"%s/%s", default_log_prefix, log_base_dir);
341 static bool InitFilestoreDirectory(
const char *dir)
343 const uint8_t dir_count = 0xff;
346 SCLogInfo(
"Filestore (v2) creating directory %s", dir);
349 "Filestore (v2) failed to create directory %s: %s", dir,
355 for (
int i = 0; i <= dir_count; i++) {
357 int n = snprintf(leaf,
sizeof(leaf),
"%s/%02x", dir, i);
358 if (n < 0 || n >= PATH_MAX) {
360 "Filestore (v2) failed to create leaf directory: " 365 SCLogInfo(
"Filestore (v2) creating directory %s", leaf);
368 "Filestore (v2) failed to create directory %s: %s",
369 leaf, strerror(errno));
376 char tmpdir[PATH_MAX];
377 int n = snprintf(tmpdir,
sizeof(tmpdir),
"%s/tmp", dir);
378 if (n < 0 || n >= PATH_MAX) {
380 "Filestore (v2) failed to create tmp directory: path too long");
384 SCLogInfo(
"Filestore (v2) creating directory %s", tmpdir);
387 "Filestore (v2) failed to create directory %s: %s", tmpdir,
412 "A file data logger is already enabled. Filestore (v2) " 413 "will not be enabled.");
417 char log_directory[PATH_MAX] =
"";
418 GetLogDirectory(conf, log_directory,
sizeof(log_directory));
419 if (!InitFilestoreDirectory(log_directory)) {
423 OutputFilestoreCtx *ctx =
SCCalloc(1,
sizeof(*ctx));
428 strlcpy(ctx->prefix, log_directory,
sizeof(ctx->prefix));
429 int written = snprintf(ctx->tmpdir,
sizeof(ctx->tmpdir) - 1,
"%s/tmp",
431 if (written ==
sizeof(ctx->tmpdir)) {
438 if (ctx->xff_cfg != NULL) {
448 output_ctx->
data = ctx;
449 output_ctx->
DeInit = OutputFilestoreLogDeInitCtx;
453 if (write_fileinfo != NULL &&
ConfValIsTrue(write_fileinfo)) {
454 #ifdef HAVE_LIBJANSSON 455 SCLogConfig(
"Filestore (v2) will output fileinfo records.");
456 ctx->fileinfo =
true;
459 "Filestore (v2) requires JSON support to log fileinfo records");
465 if (force_filestore != NULL &&
ConfValIsTrue(force_filestore)) {
467 SCLogInfo(
"forcing filestore of all files");
473 SCLogConfig(
"Filestore (v2) forcing magic lookup for stored files");
483 if (stream_depth_str != NULL && strcmp(stream_depth_str,
"no")) {
484 uint32_t stream_depth = 0;
486 &stream_depth) < 0) {
488 "file-store.stream-depth " 489 "from conf file - %s. Killing engine",
499 if (file_count_str != NULL) {
500 uint32_t file_count = 0;
504 "file-store.max-open-files " 505 "from conf file - %s. Killing engine",
509 if (file_count != 0) {
510 FileSetMaxOpenFiles(file_count);
511 SCLogConfig(
"Filestore (v2) will keep a max of %d " 512 "simultaneously open files", file_count);
518 OutputFilestoreOpenFilesCounter);
520 result.
ctx = output_ctx;
531 OutputFilestoreLogInitCtx, OutputFilestoreLogger,
532 OutputFilestoreLogThreadInit, OutputFilestoreLogThreadDeinit,
#define OUTPUT_FILEDATA_FLAG_CLOSE
#define SC_ATOMIC_DECLARE(type, name)
wrapper to declare an atomic variable including a (spin) lock to protect it.
size_t strlcpy(char *dst, const char *src, size_t siz)
void HttpXFFGetCfg(ConfNode *conf, HttpXFFCfg *result)
Function to return XFF configuration from a configuration node.
void OutputFilestoreRegister(void)
#define SC_ATOMIC_ADD(name, val)
add a value to our atomic variable
void(* DeInit)(struct OutputCtx_ *)
#define SC_ATOMIC_SUB(name, val)
sub a value from our atomic variable
bool SCPathExists(const char *path)
Check if a path exists.
void FileReassemblyDepthEnable(uint32_t size)
#define SCReturnCT(x, type)
uint16_t StatsRegisterCounter(const char *name, struct ThreadVars_ *tv)
Registers a normal, unqualified counter.
#define SC_ATOMIC_INIT(name)
Initialize the previously declared atomic variable and it's lock.
const char * ConfNodeLookupChildValue(const ConfNode *node, const char *name)
Lookup the value of a child configuration node by name.
#define SCLogError(err_code,...)
Macro used to log ERROR messages.
void FileForceHashParseCfg(ConfNode *conf)
Function to parse forced file hashing configuration.
void FileForceMagicEnable(void)
void FileForceFilestoreEnable(void)
void StatsIncr(ThreadVars *tv, uint16_t id)
Increments the local counter.
int PathIsAbsolute(const char *path)
Check if a path is absolute.
uint16_t StatsRegisterGlobalCounter(const char *name, uint64_t(*Func)(void))
Registers a counter, which represents a global value.
#define SCLogWarning(err_code,...)
Macro used to log WARNING messages.
int ConfValIsTrue(const char *val)
Check if a value is true.
const char * ConfigGetLogDirectory()
#define SC_ATOMIC_SET(name, val)
Set the value for the atomic variable.
void FileForceSha256Enable(void)
int RunModeOutputFiledataEnabled(void)
#define SCLogInfo(...)
Macro used to log INFORMATIONAL messages.
void OutputRegisterFiledataModule(LoggerId id, const char *name, const char *conf_name, OutputInitFunc InitFunc, FiledataLogger FiledataLogFunc, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, ThreadExitPrintStatsFunc ThreadExitPrintStats)
Register a file data output module.
#define SC_ATOMIC_GET(name)
Get the value from the atomic variable.
int ConfGetChildValueInt(const ConfNode *base, const char *name, intmax_t *val)
int SCCreateDirectoryTree(const char *path, const bool final)
Recursively create a directory.
Per thread variable structure.
int ParseSizeStringU32(const char *size, uint32_t *res)
struct SCLogConfig_ SCLogConfig
Holds the config state used by the logging api.
int SCDefaultMkDir(const char *path)
Wrapper around SCMkDir with default mode arguments.
#define OUTPUT_FILEDATA_FLAG_OPEN