33 #define MODULE_NAME "OutputFilestore"
37 #define SHA256_STRING_LEN (SC_SHA256_LEN * 2)
38 #define LEAF_DIR_MAX_LEN 4
39 #define FILESTORE_PREFIX_MAX (PATH_MAX - SHA256_STRING_LEN - LEAF_DIR_MAX_LEN)
43 static const char *default_log_dir =
"filestore";
73 static thread_local
bool once_errs[
WOT_MAX];
75 #define WARN_ONCE(wot_type, ...) \
77 if (!once_errs[wot_type]) { \
78 once_errs[wot_type] = true; \
79 SCLogWarning(__VA_ARGS__); \
83 static uint64_t OutputFilestoreOpenFilesCounter(
void)
88 static uint32_t g_file_store_max_open_files = 0;
90 static void FileSetMaxOpenFiles(uint32_t count)
92 g_file_store_max_open_files = count;
95 static uint32_t FileGetMaxOpenFiles(
void)
97 return g_file_store_max_open_files;
107 static void OutputFilestoreUpdateFileTime(
const char *src_filename,
const char *filename)
110 if (stat(src_filename, &sb) != 0) {
111 SCLogDebug(
"Failed to stat %s: %s", filename, strerror(errno));
114 struct utimbuf utimbuf = {
115 .actime = sb.st_atime,
116 .modtime = sb.st_mtime,
118 if (utime(filename, &utimbuf) != 0) {
119 SCLogDebug(
"Failed to update file timestamps: %s: %s", filename, strerror(errno));
133 char tmp_filename[PATH_MAX] =
"";
134 char tmp_filepath[PATH_MAX] =
"";
135 snprintf(tmp_filename,
sizeof(tmp_filename),
"file.%u", ff->
file_store_id);
136 if (
PathMerge(tmp_filepath,
sizeof(tmp_filepath),
ctx->tmpdir, tmp_filename) < 0)
140 char final_sha_short[3];
141 snprintf(final_sha_short,
sizeof(final_sha_short),
"%c%c", sha256string[0], sha256string[1]);
142 char final_sha_portion[PATH_MAX];
143 if (
PathMerge(final_sha_portion,
sizeof(final_sha_portion), final_sha_short, sha256string) < 0)
145 char final_filename[PATH_MAX] =
"";
146 if (
PathMerge(final_filename,
sizeof(final_filename),
ctx->prefix, final_sha_portion) < 0)
150 OutputFilestoreUpdateFileTime(tmp_filepath, final_filename);
151 if (unlink(tmp_filepath) != 0) {
156 }
else if (rename(tmp_filepath, final_filename) != 0) {
160 if (unlink(tmp_filepath) != 0) {
169 char js_metadata_filename_suffix[PATH_MAX];
170 if (snprintf(js_metadata_filename_suffix,
sizeof(js_metadata_filename_suffix),
172 ff->
file_store_id) == (
int)
sizeof(js_metadata_filename_suffix)) {
176 char js_metadata_filename[PATH_MAX];
177 strlcpy(js_metadata_filename, final_filename,
sizeof(js_metadata_filename));
178 strlcat(js_metadata_filename, js_metadata_filename_suffix,
sizeof(js_metadata_filename));
180 SCJsonBuilder *js_fileinfo =
182 if (
likely(js_fileinfo != NULL)) {
183 SCJbClose(js_fileinfo);
184 FILE *out = fopen(js_metadata_filename,
"w");
186 size_t js_len = SCJbLen(js_fileinfo);
187 fwrite(SCJbPtr(js_fileinfo), js_len, 1, out);
190 SCJbFree(js_fileinfo);
196 void *tx,
const uint64_t tx_id,
const uint8_t *data, uint32_t data_len, uint8_t
flags,
202 char filename[PATH_MAX] =
"";
205 SCLogDebug(
"ff %p, data %p, data_len %u", ff, data, data_len);
209 char tmp_filename[PATH_MAX] =
"";
210 snprintf(tmp_filename,
sizeof(tmp_filename),
"file.%u", ff->
file_store_id);
211 if (
PathMerge(filename,
sizeof(filename),
ctx->tmpdir, tmp_filename) < 0)
214 file_fd = open(filename, O_CREAT | O_TRUNC |
O_NOFOLLOW | O_WRONLY, 0644);
217 SCLogWarning(
"Filestore (v2) failed to create %s: %s", filename, strerror(errno));
221 if (
SC_ATOMIC_GET(filestore_open_file_cnt) < FileGetMaxOpenFiles()) {
225 if (FileGetMaxOpenFiles() > 0) {
231 }
else if (data != NULL) {
234 char tmp_filename[PATH_MAX] =
"";
235 snprintf(tmp_filename,
sizeof(tmp_filename),
"file.%u", ff->
file_store_id);
236 if (
PathMerge(filename,
sizeof(filename),
ctx->tmpdir, tmp_filename) < 0)
238 file_fd = open(filename, O_APPEND |
O_NOFOLLOW | O_WRONLY);
251 ssize_t r = write(file_fd, (
const void *)data, (
size_t)data_len);
254 char tmp_filename[PATH_MAX] =
"";
255 snprintf(tmp_filename,
sizeof(tmp_filename),
"file.%u", ff->
file_store_id);
256 (void)
PathMerge(filename,
sizeof(filename),
ctx->tmpdir, tmp_filename);
276 OutputFilestoreFinalizeFiles(
tv, aft,
ctx, p, ff, tx, tx_id, dir);
282 static TmEcode OutputFilestoreLogThreadInit(
ThreadVars *t,
const void *initdata,
void **data)
288 if (initdata == NULL) {
289 SCLogDebug(
"Error getting context for LogFileStore. \"initdata\" argument NULL");
322 static void OutputFilestoreLogDeInitCtx(
OutputCtx *output_ctx)
325 if (
ctx->xff_cfg != NULL) {
332 static int GetLogDirectory(
const SCConfNode *conf,
char *out,
size_t out_size)
335 if (log_base_dir == NULL) {
336 SCLogConfig(
"Filestore (v2) default log directory %s", default_log_dir);
337 log_base_dir = default_log_dir;
340 size_t r =
strlcpy(out, log_base_dir, out_size);
345 if (
PathMerge(out, out_size, default_log_prefix, log_base_dir) < 0)
351 static bool InitFilestoreDirectory(
const char *dir)
353 const uint8_t dir_count = 0xff;
356 SCLogInfo(
"Filestore (v2) creating directory %s", dir);
358 SCLogError(
"Filestore (v2) failed to create directory %s: %s", dir, strerror(errno));
363 for (
int i = 0; i <= dir_count; i++) {
365 snprintf(hex,
sizeof(hex),
"%02x", i);
367 if (
PathMerge(leaf,
sizeof(leaf), dir, hex) < 0) {
368 SCLogError(
"Filestore (v2) failed to create leaf directory: "
373 SCLogInfo(
"Filestore (v2) creating directory %s", leaf);
376 "Filestore (v2) failed to create directory %s: %s", leaf, strerror(errno));
383 char tmpdir[PATH_MAX];
384 if (
PathMerge(tmpdir,
sizeof(tmpdir), dir,
"tmp") < 0) {
385 SCLogError(
"Filestore (v2) failed to create tmp directory: path too long");
389 SCLogInfo(
"Filestore (v2) creating directory %s", tmpdir);
391 SCLogError(
"Filestore (v2) failed to create directory %s: %s", tmpdir, strerror(errno));
409 SCLogWarning(
"File-store v1 has been removed. Please update to file-store v2.");
414 SCLogWarning(
"A file data logger is already enabled. Filestore (v2) "
415 "will not be enabled.");
419 char log_directory[PATH_MAX] =
"";
420 if (GetLogDirectory(conf, log_directory,
sizeof(log_directory)) < 0) {
421 SCLogError(
"File-store output path too long.");
424 if (!InitFilestoreDirectory(log_directory)) {
435 if (
PathMerge(
ctx->tmpdir,
sizeof(
ctx->tmpdir), log_directory,
"tmp") < 0) {
436 SCLogError(
"File-store output directory overflow.");
442 if (
ctx->xff_cfg != NULL) {
454 output_ctx->
DeInit = OutputFilestoreLogDeInitCtx;
458 SCLogConfig(
"Filestore (v2) will output fileinfo records.");
459 ctx->fileinfo =
true;
465 SCLogInfo(
"forcing filestore of all files");
471 SCLogConfig(
"Filestore (v2) forcing magic lookup for stored files");
482 if (stream_depth_str != NULL && strcmp(stream_depth_str,
"no")) {
483 uint32_t stream_depth = 0;
486 "file-store.stream-depth "
487 "from conf file - %s. Killing engine",
493 SCLogWarning(
"file-store.stream-depth value %" PRIu32
" has "
494 "no effect since it's less than stream.reassembly.depth "
504 if (file_count_str != NULL) {
505 uint32_t file_count = 0;
508 "file-store.max-open-files "
509 "from conf file - %s. Killing engine",
513 if (file_count != 0) {
514 FileSetMaxOpenFiles(file_count);
515 SCLogConfig(
"Filestore (v2) will keep a max of %d "
516 "simultaneously open files",
522 result.
ctx = output_ctx;
530 OutputFilestoreLogInitCtx, OutputFilestoreLogger, OutputFilestoreLogThreadInit,
531 OutputFilestoreLogThreadDeinit);