Skip to content
API Documentation
BastionGuard™
carving::anonymous_namespace{carving.cpp} Namespace Reference

Classes

struct  DetectResult
struct  CarverFormat

Typedefs

using PeekMore

Functions

uint16_t le16 (const unsigned char *p)
uint32_t le32 (const unsigned char *p)
uint32_t be32 (const unsigned char *p)
uint64_t be64 (const unsigned char *p)
size_t find_pattern (const unsigned char *buf, size_t buf_len, const unsigned char *pat, size_t pat_len, size_t start=0)
DetectResult detect_jpeg (const unsigned char *window, size_t window_len, uint64_t file_disk_off, uint64_t max_file_size, PeekMore &peek_more)
DetectResult detect_png (const unsigned char *window, size_t window_len, uint64_t file_disk_off, uint64_t max_file_size, PeekMore &peek_more)
DetectResult detect_gif (const unsigned char *window, size_t window_len, uint64_t file_disk_off, uint64_t max_file_size, PeekMore &peek_more)
DetectResult detect_pdf (const unsigned char *window, size_t window_len, uint64_t file_disk_off, uint64_t max_file_size, PeekMore &peek_more)
DetectResult detect_zip (const unsigned char *window, size_t window_len, uint64_t file_disk_off, uint64_t max_file_size, PeekMore &peek_more)
DetectResult detect_bmp (const unsigned char *window, size_t window_len, uint64_t, uint64_t max_file_size, PeekMore &)
DetectResult detect_mp4 (const unsigned char *window, size_t window_len, uint64_t file_disk_off, uint64_t max_file_size, PeekMore &peek_more)
DetectResult detect_mp3_id3 (const unsigned char *window, size_t window_len, uint64_t, uint64_t max_file_size, PeekMore &)
DetectResult detect_mp3_frame (const unsigned char *window, size_t window_len, uint64_t, uint64_t max_file_size, PeekMore &)
DetectResult detect_ogg (const unsigned char *window, size_t window_len, uint64_t file_disk_off, uint64_t max_file_size, PeekMore &peek_more)
DetectResult detect_tiff (const unsigned char *window, size_t window_len, uint64_t, uint64_t max_file_size, PeekMore &)
DetectResult detect_ole (const unsigned char *window, size_t window_len, uint64_t, uint64_t max_file_size, PeekMore &)
DetectResult detect_riff (const unsigned char *window, size_t window_len, uint64_t, uint64_t max_file_size, PeekMore &)
DetectResult detect_mkv (const unsigned char *window, size_t window_len, uint64_t, uint64_t max_file_size, PeekMore &)
DetectResult detect_sqlite (const unsigned char *window, size_t window_len, uint64_t, uint64_t max_file_size, PeekMore &)
std::vector< CarverFormat > build_formats ()

Typedef Documentation

◆ PeekMore

using carving::anonymous_namespace{carving.cpp}::PeekMore
Initial value:
std::function<size_t(uint64_t abs_off, unsigned char* dst,
size_t want)>

Definition at line 77 of file carving.cpp.

Function Documentation

◆ be32()

uint32_t carving::anonymous_namespace{carving.cpp}::be32 ( const unsigned char * p)
inline

Definition at line 36 of file carving.cpp.

36 {
37 return (static_cast<uint32_t>(p[0]) << 24) |
38 (static_cast<uint32_t>(p[1]) << 16) |
39 (static_cast<uint32_t>(p[2]) << 8) |
40 static_cast<uint32_t>(p[3]);
41}

References be32().

Referenced by be32(), detect_mp4(), detect_png(), and detect_tiff().

◆ be64()

uint64_t carving::anonymous_namespace{carving.cpp}::be64 ( const unsigned char * p)
inline

Definition at line 42 of file carving.cpp.

42 {
43 return (static_cast<uint64_t>(p[0]) << 56) |
44 (static_cast<uint64_t>(p[1]) << 48) |
45 (static_cast<uint64_t>(p[2]) << 40) |
46 (static_cast<uint64_t>(p[3]) << 32) |
47 (static_cast<uint64_t>(p[4]) << 24) |
48 (static_cast<uint64_t>(p[5]) << 16) |
49 (static_cast<uint64_t>(p[6]) << 8) |
50 static_cast<uint64_t>(p[7]);
51}

References be64().

Referenced by be64(), and detect_mp4().

◆ build_formats()

std::vector< CarverFormat > carving::anonymous_namespace{carving.cpp}::build_formats ( )

Definition at line 899 of file carving.cpp.

899 {
900 std::vector<CarverFormat> v;
901 v.push_back({"jpeg", "jpg",
902 {0xFF, 0xD8, 0xFF}, 0, &detect_jpeg});
903 v.push_back({"png", "png",
904 {0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}, 0,
905 &detect_png});
906 v.push_back({"gif", "gif",
907 {'G','I','F','8','9','a'}, 0, &detect_gif});
908 v.push_back({"gif87","gif",
909 {'G','I','F','8','7','a'}, 0, &detect_gif});
910 v.push_back({"pdf", "pdf",
911 {'%','P','D','F','-'}, 0, &detect_pdf});
912 v.push_back({"zip", "zip",
913 {'P','K',0x03,0x04}, 0, &detect_zip});
914 v.push_back({"bmp", "bmp",
915 {'B','M'}, 0, &detect_bmp});
916 // RIFF unified (WAV / AVI / WEBP). Extension is set generically;
917 // in a future version we could inspect the form type to pick a
918 // better extension (.wav vs .avi vs .webp).
919 v.push_back({"riff", "riff",
920 {'R','I','F','F'}, 0, &detect_riff});
921 v.push_back({"mp4", "mp4",
922 {'f','t','y','p'}, 4, &detect_mp4});
923 // ---- New formats added in this package ----
924 v.push_back({"mp3-id3", "mp3",
925 {'I','D','3'}, 0, &detect_mp3_id3});
926 // MP3 frame-sync pairs: try the most common (MPEG-1 Layer III) only,
927 // 0xFF 0xFB. Other variants exist but the false-positive rate
928 // gets too high if we accept all of 0xFF 0xFx.
929 v.push_back({"mp3-frame", "mp3",
930 {0xFF, 0xFB}, 0, &detect_mp3_frame});
931 v.push_back({"mp3-frame-mpeg2", "mp3",
932 {0xFF, 0xF3}, 0, &detect_mp3_frame});
933 v.push_back({"ogg", "ogg",
934 {'O','g','g','S'}, 0, &detect_ogg});
935 v.push_back({"tiff-le", "tif",
936 {'I','I',0x2A,0x00}, 0, &detect_tiff});
937 v.push_back({"tiff-be", "tif",
938 {'M','M',0x00,0x2A}, 0, &detect_tiff});
939 // OLE Compound File (DOC, XLS, PPT old, MSI)
940 v.push_back({"ole-cf", "ole",
941 {0xD0,0xCF,0x11,0xE0,0xA1,0xB1,0x1A,0xE1}, 0,
942 &detect_ole});
943 // Note: AVI is handled by the unified detect_riff above
944 v.push_back({"mkv", "mkv",
945 {0x1A,0x45,0xDF,0xA3}, 0, &detect_mkv});
946 v.push_back({"sqlite", "sqlite",
947 {'S','Q','L','i','t','e',' ','f','o','r','m','a','t',
948 ' ','3',0x00}, 0, &detect_sqlite});
949 return v;
950}
carving::anonymous_namespace{carving.cpp}::detect_riff
DetectResult detect_riff(const unsigned char *window, size_t window_len, uint64_t, uint64_t max_file_size, PeekMore &)
Definition carving.cpp:799
carving::anonymous_namespace{carving.cpp}::detect_gif
DetectResult detect_gif(const unsigned char *window, size_t window_len, uint64_t file_disk_off, uint64_t max_file_size, PeekMore &peek_more)
Definition carving.cpp:258
carving::anonymous_namespace{carving.cpp}::detect_jpeg
DetectResult detect_jpeg(const unsigned char *window, size_t window_len, uint64_t file_disk_off, uint64_t max_file_size, PeekMore &peek_more)
Definition carving.cpp:113
carving::anonymous_namespace{carving.cpp}::detect_mkv
DetectResult detect_mkv(const unsigned char *window, size_t window_len, uint64_t, uint64_t max_file_size, PeekMore &)
Definition carving.cpp:835
carving::anonymous_namespace{carving.cpp}::detect_pdf
DetectResult detect_pdf(const unsigned char *window, size_t window_len, uint64_t file_disk_off, uint64_t max_file_size, PeekMore &peek_more)
Definition carving.cpp:313
carving::anonymous_namespace{carving.cpp}::detect_ogg
DetectResult detect_ogg(const unsigned char *window, size_t window_len, uint64_t file_disk_off, uint64_t max_file_size, PeekMore &peek_more)
Definition carving.cpp:671
carving::anonymous_namespace{carving.cpp}::detect_tiff
DetectResult detect_tiff(const unsigned char *window, size_t window_len, uint64_t, uint64_t max_file_size, PeekMore &)
Definition carving.cpp:745
carving::anonymous_namespace{carving.cpp}::detect_ole
DetectResult detect_ole(const unsigned char *window, size_t window_len, uint64_t, uint64_t max_file_size, PeekMore &)
Definition carving.cpp:772
carving::anonymous_namespace{carving.cpp}::detect_zip
DetectResult detect_zip(const unsigned char *window, size_t window_len, uint64_t file_disk_off, uint64_t max_file_size, PeekMore &peek_more)
Definition carving.cpp:397
carving::anonymous_namespace{carving.cpp}::detect_bmp
DetectResult detect_bmp(const unsigned char *window, size_t window_len, uint64_t, uint64_t max_file_size, PeekMore &)
Definition carving.cpp:489
carving::anonymous_namespace{carving.cpp}::detect_mp3_frame
DetectResult detect_mp3_frame(const unsigned char *window, size_t window_len, uint64_t, uint64_t max_file_size, PeekMore &)
Definition carving.cpp:637
carving::anonymous_namespace{carving.cpp}::detect_mp4
DetectResult detect_mp4(const unsigned char *window, size_t window_len, uint64_t file_disk_off, uint64_t max_file_size, PeekMore &peek_more)
Definition carving.cpp:526
carving::anonymous_namespace{carving.cpp}::detect_mp3_id3
DetectResult detect_mp3_id3(const unsigned char *window, size_t window_len, uint64_t, uint64_t max_file_size, PeekMore &)
Definition carving.cpp:605
carving::anonymous_namespace{carving.cpp}::detect_png
DetectResult detect_png(const unsigned char *window, size_t window_len, uint64_t file_disk_off, uint64_t max_file_size, PeekMore &peek_more)
Definition carving.cpp:187
carving::anonymous_namespace{carving.cpp}::detect_sqlite
DetectResult detect_sqlite(const unsigned char *window, size_t window_len, uint64_t, uint64_t max_file_size, PeekMore &)
Definition carving.cpp:859

References build_formats(), detect_bmp(), detect_gif(), detect_jpeg(), detect_mkv(), detect_mp3_frame(), detect_mp3_id3(), detect_mp4(), detect_ogg(), detect_ole(), detect_pdf(), detect_png(), detect_riff(), detect_sqlite(), detect_tiff(), and detect_zip().

Referenced by build_formats().

◆ detect_bmp()

DetectResult carving::anonymous_namespace{carving.cpp}::detect_bmp ( const unsigned char * window,
size_t window_len,
uint64_t ,
uint64_t max_file_size,
PeekMore &  )

Definition at line 489 of file carving.cpp.

492 {
493 DetectResult r;
494 if (window_len < 14) return r;
495 const uint32_t fsz = le32(window + 2);
496 // DIB header size at offset 14 must be one of the standard values
497 // (12, 40, 52, 56, 64, 108, 124).
498 if (window_len < 18) return r;
499 const uint32_t dib_size = le32(window + 14);
500 static const uint32_t valid_dib[] = {12, 40, 52, 56, 64, 108, 124};
501 bool dib_ok = false;
502 for (uint32_t v : valid_dib) if (dib_size == v) { dib_ok = true; break; }
503 if (!dib_ok) return r;
504 if (fsz < 26 || fsz > max_file_size) {
505 if (fsz > max_file_size) {
506 r.ok = true;
507 r.size = max_file_size;
508 r.size_is_capped = true;
509 return r;
510 }
511 return r;
512 }
513 r.ok = true;
514 r.size = fsz;
515 return r;
516}
carving::anonymous_namespace{carving.cpp}::le32
uint32_t le32(const unsigned char *p)
Definition carving.cpp:30
make_ads_rules.r
r
Definition make_ads_rules.py:20
carving::anonymous_namespace{carving.cpp}::DetectResult
Definition carving.cpp:80

References detect_bmp(), and le32().

Referenced by build_formats(), and detect_bmp().

◆ detect_gif()

DetectResult carving::anonymous_namespace{carving.cpp}::detect_gif ( const unsigned char * window,
size_t window_len,
uint64_t file_disk_off,
uint64_t max_file_size,
PeekMore & peek_more )

Definition at line 258 of file carving.cpp.

261 {
262 DetectResult r;
263 // Validate the logical screen descriptor immediately follows the magic
264 if (window_len < 13) return r;
265 // The byte at offset 10 is the packed field; bit 7 = global color
266 // table flag, bits 0-2 = color resolution (must be 0..7). We don't
267 // strictly validate; just sanity-check that bytes 10..12 are not
268 // all the same (a degenerate value is suspicious).
269 if (window[10] == window[11] && window[11] == window[12]
270 && (window[10] == 0x00 || window[10] == 0xFF)) return r;
271
272 // Scan for trailer 0x3B
273 constexpr size_t SCAN_CHUNK = 256 * 1024;
274 std::vector<unsigned char> chunk(SCAN_CHUNK);
275 constexpr unsigned char trailer = 0x3B;
276
277 // Existing window first
278 for (size_t i = 13; i < window_len; ++i) {
279 if (window[i] == trailer && i > 0 && window[i - 1] == 0x00) {
280 r.ok = true;
281 r.size = i + 1;
282 return r;
283 }
284 }
285 uint64_t total = window_len;
286 uint64_t cursor = file_disk_off + window_len;
287 unsigned char prev_last = window_len > 0 ? window[window_len - 1] : 0;
288 while (total < max_file_size) {
289 size_t got = peek_more(cursor, chunk.data(), SCAN_CHUNK);
290 if (got == 0) break;
291 for (size_t i = 0; i < got; ++i) {
292 const unsigned char prev = (i == 0) ? prev_last : chunk[i - 1];
293 if (chunk[i] == trailer && prev == 0x00) {
294 r.ok = true;
295 r.size = total + i + 1;
296 return r;
297 }
298 }
299 prev_last = chunk[got - 1];
300 total += got;
301 cursor += got;
302 }
303 return r;
304}

References detect_gif().

Referenced by build_formats(), and detect_gif().

◆ detect_jpeg()

DetectResult carving::anonymous_namespace{carving.cpp}::detect_jpeg ( const unsigned char * window,
size_t window_len,
uint64_t file_disk_off,
uint64_t max_file_size,
PeekMore & peek_more )

Definition at line 113 of file carving.cpp.

116 {
117 DetectResult r;
118 if (window_len < 4) return r;
119 // The 4th byte must be a valid JPEG marker (0xE0..0xEF for APP,
120 // or 0xDB for DQT, or 0xC0..0xCF for SOF, or 0xFE for COM).
121 const uint8_t b3 = window[3];
122 const bool good_marker =
123 (b3 >= 0xE0 && b3 <= 0xEF) || // APP0..APPF
124 b3 == 0xDB || // DQT
125 (b3 >= 0xC0 && b3 <= 0xCF) || // SOF
126 b3 == 0xFE; // COM
127 if (!good_marker) return r;
128
129 // Search for FF D9 footer. Read up to max_file_size in 1 MiB
130 // chunks, scanning each for the footer.
131 constexpr size_t SCAN_CHUNK = 1024 * 1024;
132 constexpr unsigned char footer[2] = { 0xFF, 0xD9 };
133
134 // Start by scanning the existing window
135 size_t hit = find_pattern(window, window_len, footer, 2, 4);
136 if (hit != SIZE_MAX) {
137 r.ok = true;
138 r.size = hit + 2;
139 return r;
140 }
141
142 // Read more from disk
143 std::vector<unsigned char> chunk(SCAN_CHUNK);
144 uint64_t cursor = file_disk_off + window_len;
145 uint64_t total = window_len;
146 // Carry one byte across chunks in case footer straddles boundary
147 bool have_carry = (window_len > 0 && window[window_len - 1] == 0xFF);
148
149 while (total < max_file_size) {
150 size_t got = peek_more(cursor, chunk.data(), SCAN_CHUNK);
151 if (got == 0) break;
152
153 // Boundary-straddle check
154 if (have_carry && got >= 1 && chunk[0] == 0xD9) {
155 r.ok = true;
156 r.size = total + 1;
157 return r;
158 }
159
160 size_t local = find_pattern(chunk.data(), got, footer, 2);
161 if (local != SIZE_MAX) {
162 r.ok = true;
163 r.size = total + local + 2;
164 return r;
165 }
166 have_carry = (got > 0 && chunk[got - 1] == 0xFF);
167 total += got;
168 cursor += got;
169 }
170 // No footer found within max_file_size -- soft-cap at max
171 if (total >= max_file_size) {
172 r.ok = true;
173 r.size = max_file_size;
174 r.size_is_capped = true;
175 return r;
176 }
177 return r;
178}
carving::anonymous_namespace{carving.cpp}::find_pattern
size_t find_pattern(const unsigned char *buf, size_t buf_len, const unsigned char *pat, size_t pat_len, size_t start=0)
Definition carving.cpp:55

References detect_jpeg(), and find_pattern().

Referenced by build_formats(), and detect_jpeg().

◆ detect_mkv()

DetectResult carving::anonymous_namespace{carving.cpp}::detect_mkv ( const unsigned char * window,
size_t window_len,
uint64_t ,
uint64_t max_file_size,
PeekMore &  )

Definition at line 835 of file carving.cpp.

838 {
839 DetectResult r;
840 if (window_len < 16) return r;
841 // After the 4-byte EBML header magic, the next byte starts the
842 // EBML header VINT. The first byte should have at least one set
843 // bit indicating length; common is 0x9F for a small EBML header.
844 // Filter out clearly-bogus follow-ups.
845 if (window[4] == 0x00) return r;
846 constexpr uint64_t MKV_HARD_CAP = 4ull * 1024 * 1024 * 1024; // 4 GB
847 r.ok = true;
848 r.size = std::min<uint64_t>(MKV_HARD_CAP, max_file_size);
849 r.size_is_capped = true;
850 return r;
851}

References detect_mkv().

Referenced by build_formats(), and detect_mkv().

◆ detect_mp3_frame()

DetectResult carving::anonymous_namespace{carving.cpp}::detect_mp3_frame ( const unsigned char * window,
size_t window_len,
uint64_t ,
uint64_t max_file_size,
PeekMore &  )

Definition at line 637 of file carving.cpp.

640 {
641 DetectResult r;
642 if (window_len < 4) return r;
643 // 11-bit frame sync (0xFFE..0xFFF) followed by version (2 bits, must
644 // be 00 or 11 -- 01 reserved), layer (2 bits, 01..11 -- 00 reserved),
645 // protection bit. We accept only V1/V2/V2.5 + Layer III commonly.
646 const uint8_t b1 = window[1];
647 const uint8_t b2 = window[2];
648 if ((b1 & 0xE0) != 0xE0) return r;
649 const uint8_t version = (b1 >> 3) & 0x03;
650 const uint8_t layer = (b1 >> 1) & 0x03;
651 if (version == 0x01) return r; // reserved
652 if (layer == 0x00) return r; // reserved
653 const uint8_t bitrate_idx = (b2 >> 4) & 0x0F;
654 const uint8_t sample_idx = (b2 >> 2) & 0x03;
655 if (bitrate_idx == 0x0F || sample_idx == 0x03) return r; // bad
656 // We don't walk frames. Use a heuristic file-size cap.
657 constexpr uint64_t MP3_HARD_CAP = 30ull * 1024 * 1024;
658 r.ok = true;
659 r.size = std::min<uint64_t>(MP3_HARD_CAP, max_file_size);
660 r.size_is_capped = true;
661 return r;
662}

References detect_mp3_frame().

Referenced by build_formats(), and detect_mp3_frame().

◆ detect_mp3_id3()

DetectResult carving::anonymous_namespace{carving.cpp}::detect_mp3_id3 ( const unsigned char * window,
size_t window_len,
uint64_t ,
uint64_t max_file_size,
PeekMore &  )

Definition at line 605 of file carving.cpp.

608 {
609 DetectResult r;
610 if (window_len < 10) return r;
611 // ID3v2 header layout (10 bytes):
612 // "ID3" (3)
613 // version (2)
614 // flags (1)
615 // size (4 bytes, syncsafe: each byte uses 7 bits)
616 if (window[3] == 0xFF || window[4] == 0xFF) return r; // implausible version
617 // Syncsafe size decoding
618 const uint32_t s0 = window[6] & 0x7F;
619 const uint32_t s1 = window[7] & 0x7F;
620 const uint32_t s2 = window[8] & 0x7F;
621 const uint32_t s3 = window[9] & 0x7F;
622 const uint32_t tag_size = (s0 << 21) | (s1 << 14) | (s2 << 7) | s3;
623 // Total ID3v2 prefix = 10 + tag_size. After that should come MP3
624 // frames. We don't try to walk frames -- we assume the whole file
625 // is one MP3 stream and use a heuristic cap.
626 const uint64_t prefix = 10 + tag_size;
627 // Most MP3 files are 1-15 MB; cap at 30 MB for safety.
628 constexpr uint64_t MP3_HARD_CAP = 30ull * 1024 * 1024;
629 const uint64_t cap = std::min<uint64_t>(MP3_HARD_CAP, max_file_size);
630 if (prefix > cap) return r;
631 r.ok = true;
632 r.size = cap;
633 r.size_is_capped = true;
634 return r;
635}

References detect_mp3_id3().

Referenced by build_formats(), and detect_mp3_id3().

◆ detect_mp4()

DetectResult carving::anonymous_namespace{carving.cpp}::detect_mp4 ( const unsigned char * window,
size_t window_len,
uint64_t file_disk_off,
uint64_t max_file_size,
PeekMore & peek_more )

Definition at line 526 of file carving.cpp.

529 {
530 DetectResult r;
531 if (window_len < 16) return r;
532 // First atom: size (4 BE) + 'ftyp' (4)
533 const uint32_t atom1_size = be32(window);
534 if (atom1_size < 8 || atom1_size > 1024) return r;
535 if (std::memcmp(window + 4, "ftyp", 4) != 0) return r;
536
537 // The major brand at offset 8..12 should be ASCII alphanumeric
538 // (e.g. "isom", "mp42", "qt ", "M4A ", "M4V ", "iso2"). Filter
539 // junk.
540 bool brand_ok = true;
541 for (int i = 0; i < 4; ++i) {
542 const unsigned char c = window[8 + i];
543 if (!(c == ' ' || (c >= '0' && c <= '9') ||
544 (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'))) {
545 brand_ok = false;
546 break;
547 }
548 }
549 if (!brand_ok) return r;
550
551 // Walk atoms
552 constexpr int MAX_ATOMS = 10000;
553 uint64_t pos = 0;
554 for (int n = 0; n < MAX_ATOMS; ++n) {
555 if (pos + 8 > max_file_size) {
556 r.ok = true;
557 r.size = max_file_size;
558 r.size_is_capped = true;
559 return r;
560 }
561 unsigned char hdr[16];
562 // Read atom header
563 size_t got = 0;
564 if (pos + 8 <= window_len) {
565 std::memcpy(hdr, window + pos, 8);
566 got = 8;
567 } else {
568 got = peek_more(file_disk_off + pos, hdr, 16);
569 }
570 if (got < 8) return r;
571 uint64_t atom_size = be32(hdr);
572 if (atom_size == 1) {
573 // Extended 64-bit size at offset 8..16
574 if (got < 16) {
575 got += peek_more(file_disk_off + pos + 8, hdr + 8, 8);
576 }
577 if (got < 16) return r;
578 atom_size = be64(hdr + 8);
579 } else if (atom_size == 0) {
580 // 0 means "to end of file" -- not useful for carving
581 r.ok = true;
582 r.size = std::min<uint64_t>(max_file_size, pos + 8);
583 return r;
584 }
585 if (atom_size < 8 || atom_size > max_file_size) return r;
586 pos += atom_size;
587 if (pos > max_file_size) {
588 r.ok = true;
589 r.size = max_file_size;
590 r.size_is_capped = true;
591 return r;
592 }
593 }
594 return r;
595}
carving::anonymous_namespace{carving.cpp}::be64
uint64_t be64(const unsigned char *p)
Definition carving.cpp:42
carving::anonymous_namespace{carving.cpp}::be32
uint32_t be32(const unsigned char *p)
Definition carving.cpp:36

References be32(), be64(), and detect_mp4().

Referenced by build_formats(), and detect_mp4().

◆ detect_ogg()

DetectResult carving::anonymous_namespace{carving.cpp}::detect_ogg ( const unsigned char * window,
size_t window_len,
uint64_t file_disk_off,
uint64_t max_file_size,
PeekMore & peek_more )

Definition at line 671 of file carving.cpp.

674 {
675 DetectResult r;
676 if (window_len < 27) return r;
677 // Stream structure version must be 0
678 if (window[4] != 0x00) return r;
679
680 auto read_at = [&](uint64_t off, unsigned char* dst, size_t want) -> size_t {
681 if (off >= file_disk_off &&
682 off + want <= file_disk_off + window_len) {
683 std::memcpy(dst, window + (off - file_disk_off), want);
684 return want;
685 }
686 return peek_more(off, dst, want);
687 };
688
689 uint64_t pos = 0;
690 constexpr int MAX_PAGES = 100000;
691 for (int n = 0; n < MAX_PAGES; ++n) {
692 if (pos + 27 > max_file_size) {
693 r.ok = true;
694 r.size = max_file_size;
695 r.size_is_capped = true;
696 return r;
697 }
698 unsigned char hdr[27];
699 if (read_at(file_disk_off + pos, hdr, 27) != 27) {
700 // EOF -- accept what we have
701 if (pos > 0) {
702 r.ok = true;
703 r.size = pos;
704 return r;
705 }
706 return r;
707 }
708 if (std::memcmp(hdr, "OggS", 4) != 0) {
709 // No more pages
710 if (pos > 0) {
711 r.ok = true;
712 r.size = pos;
713 return r;
714 }
715 return r;
716 }
717 const uint8_t header_type = hdr[5];
718 const uint8_t segment_count = hdr[26];
719 // Read segment_table
720 std::vector<unsigned char> seg_table(segment_count);
721 if (segment_count > 0 &&
722 read_at(file_disk_off + pos + 27, seg_table.data(),
723 segment_count) != segment_count) return r;
724 uint32_t body_size = 0;
725 for (uint8_t i = 0; i < segment_count; ++i) body_size += seg_table[i];
726 const uint64_t page_size = 27 + segment_count + body_size;
727 pos += page_size;
728 // header_type bit 2 = "last page of bitstream"
729 if (header_type & 0x04) {
730 r.ok = true;
731 r.size = pos;
732 return r;
733 }
734 }
735 return r;
736}

References detect_ogg().

Referenced by build_formats(), and detect_ogg().

◆ detect_ole()

DetectResult carving::anonymous_namespace{carving.cpp}::detect_ole ( const unsigned char * window,
size_t window_len,
uint64_t ,
uint64_t max_file_size,
PeekMore &  )

Definition at line 772 of file carving.cpp.

775 {
776 DetectResult r;
777 if (window_len < 76) return r;
778 // Sector shift at offset 30 (LE16): typically 9 (=512-byte sectors)
779 // or 12 (=4096-byte sectors). Other values are bogus.
780 const uint16_t sec_shift = le16(window + 30);
781 if (sec_shift != 9 && sec_shift != 12) return r;
782 // Number of sectors in directory-stream FAT chain at offset 40:
783 // a sanity bound.
784 const uint32_t num_dir_sectors = le32(window + 40);
785 if (num_dir_sectors > 100000) return r;
786 // Heuristic: 100 MB cap. Real Office files are usually <50 MB.
787 constexpr uint64_t OLE_HARD_CAP = 100ull * 1024 * 1024;
788 r.ok = true;
789 r.size = std::min<uint64_t>(OLE_HARD_CAP, max_file_size);
790 r.size_is_capped = true;
791 return r;
792}
carving::anonymous_namespace{carving.cpp}::le16
uint16_t le16(const unsigned char *p)
Definition carving.cpp:26

References detect_ole(), le16(), and le32().

Referenced by build_formats(), and detect_ole().

◆ detect_pdf()

DetectResult carving::anonymous_namespace{carving.cpp}::detect_pdf ( const unsigned char * window,
size_t window_len,
uint64_t file_disk_off,
uint64_t max_file_size,
PeekMore & peek_more )

Definition at line 313 of file carving.cpp.

316 {
317 DetectResult r;
318 // Sanity: byte at offset 5 must be a digit (PDF version "%PDF-1.x")
319 if (window_len < 8) return r;
320 if (window[5] != '1' && window[5] != '2') return r;
321 if (window[6] != '.') return r;
322
323 constexpr size_t SCAN_CHUNK = 1024 * 1024;
324 std::vector<unsigned char> chunk(SCAN_CHUNK);
325 constexpr unsigned char eof[5] = {'%', '%', 'E', 'O', 'F'};
326
327 // Last-occurrence scan: keep updating last_eof_at as we read more.
328 uint64_t last_eof_end = 0; // byte offset within file (1-past)
329
330 // Scan window first
331 for (size_t i = 0; i + 5 <= window_len; ++i) {
332 if (std::memcmp(window + i, eof, 5) == 0) {
333 last_eof_end = i + 5;
334 }
335 }
336 uint64_t total = window_len;
337 uint64_t cursor = file_disk_off + window_len;
338 // Carry the last 4 bytes across chunks for boundary handling
339 unsigned char carry[4] = {0, 0, 0, 0};
340 size_t carry_len = std::min<size_t>(4, window_len);
341 if (window_len > 0) {
342 std::memcpy(carry, window + window_len - carry_len, carry_len);
343 }
344
345 while (total < max_file_size) {
346 size_t got = peek_more(cursor, chunk.data(), SCAN_CHUNK);
347 if (got == 0) break;
348 // Build a small temp buffer (carry + first 4 bytes of chunk) for
349 // boundary scan
350 std::vector<unsigned char> bnd(carry_len + std::min<size_t>(got, 4));
351 std::memcpy(bnd.data(), carry, carry_len);
352 std::memcpy(bnd.data() + carry_len, chunk.data(),
353 std::min<size_t>(got, 4));
354 for (size_t i = 0; i + 5 <= bnd.size(); ++i) {
355 if (std::memcmp(bnd.data() + i, eof, 5) == 0) {
356 // Translate position to file offset
357 last_eof_end = total - carry_len + i + 5;
358 }
359 }
360 // Scan main chunk
361 for (size_t i = 0; i + 5 <= got; ++i) {
362 if (std::memcmp(chunk.data() + i, eof, 5) == 0) {
363 last_eof_end = total + i + 5;
364 }
365 }
366 carry_len = std::min<size_t>(4, got);
367 std::memcpy(carry, chunk.data() + got - carry_len, carry_len);
368 total += got;
369 cursor += got;
370 }
371
372 if (last_eof_end > 0) {
373 // Round up to next newline if any (PDFs commonly have %%EOF\n)
374 r.ok = true;
375 r.size = last_eof_end;
376 // Try to include trailing CR/LF (up to 2 bytes)
377 unsigned char tail[2];
378 if (peek_more(file_disk_off + last_eof_end, tail, 2) > 0) {
379 int extra = 0;
380 if (tail[0] == '\r' || tail[0] == '\n') ++extra;
381 if (extra == 1 && (tail[1] == '\r' || tail[1] == '\n')) ++extra;
382 r.size += extra;
383 }
384 if (r.size > max_file_size) { r.size = max_file_size; r.size_is_capped = true; }
385 return r;
386 }
387 return r;
388}

References detect_pdf().

Referenced by build_formats(), and detect_pdf().

◆ detect_png()

DetectResult carving::anonymous_namespace{carving.cpp}::detect_png ( const unsigned char * window,
size_t window_len,
uint64_t file_disk_off,
uint64_t max_file_size,
PeekMore & peek_more )

Definition at line 187 of file carving.cpp.

190 {
191 DetectResult r;
192 constexpr size_t HDR = 8;
193 if (window_len < HDR + 8) return r;
194
195 // First chunk MUST be IHDR (length 13, type 'IHDR')
196 const uint32_t first_len = be32(window + HDR + 0);
197 if (first_len != 13) return r;
198 if (std::memcmp(window + HDR + 4, "IHDR", 4) != 0) return r;
199
200 // Walk chunks. Each chunk is:
201 // __be32 length (length of DATA, not including these 12 hdr bytes)
202 // 4 bytes type
203 // <length> bytes data
204 // __be32 CRC
205 // Total chunk size = 12 + length.
206 constexpr size_t SCAN_CHUNK = 256 * 1024;
207 std::vector<unsigned char> chunk_buf(SCAN_CHUNK);
208
209 auto read_at = [&](uint64_t off, unsigned char* dst, size_t want) -> size_t {
210 // First try to satisfy from `window` if in range
211 if (off >= file_disk_off &&
212 off + want <= file_disk_off + window_len) {
213 std::memcpy(dst, window + (off - file_disk_off), want);
214 return want;
215 }
216 return peek_more(off, dst, want);
217 };
218
219 uint64_t pos = HDR; // file-relative offset of next chunk header
220 constexpr int MAX_CHUNKS = 100000; // sanity cap
221 for (int n = 0; n < MAX_CHUNKS; ++n) {
222 if (pos + 12 > max_file_size) {
223 r.ok = true;
224 r.size = max_file_size;
225 r.size_is_capped = true;
226 return r;
227 }
228 unsigned char hdr[12];
229 if (read_at(file_disk_off + pos, hdr, 12) != 12) return r;
230 const uint32_t clen = be32(hdr);
231 if (clen > 0x10000000u) return r; // implausible
232 const bool is_iend = std::memcmp(hdr + 4, "IEND", 4) == 0;
233 // Chunk total size = 12 (hdr+CRC) + clen
234 pos += 12 + clen;
235 if (is_iend) {
236 r.ok = true;
237 r.size = pos;
238 return r;
239 }
240 if (pos > max_file_size) {
241 r.ok = true;
242 r.size = max_file_size;
243 r.size_is_capped = true;
244 return r;
245 }
246 }
247 return r;
248}

References be32(), and detect_png().

Referenced by build_formats(), and detect_png().

◆ detect_riff()

DetectResult carving::anonymous_namespace{carving.cpp}::detect_riff ( const unsigned char * window,
size_t window_len,
uint64_t ,
uint64_t max_file_size,
PeekMore &  )

Definition at line 799 of file carving.cpp.

802 {
803 DetectResult r;
804 if (window_len < 12) return r;
805 // Check the form type at offset 8: WAVE, AVI , WEBP are accepted.
806 const bool is_wav = std::memcmp(window + 8, "WAVE", 4) == 0;
807 const bool is_avi = std::memcmp(window + 8, "AVI ", 4) == 0;
808 const bool is_webp = std::memcmp(window + 8, "WEBP", 4) == 0;
809 if (!is_wav && !is_avi && !is_webp) return r;
810 const uint32_t riff_size = le32(window + 4);
811 if (riff_size < 4) return r;
812 const uint64_t total = static_cast<uint64_t>(riff_size) + 8;
813 if (total > max_file_size) {
814 r.ok = true;
815 r.size = max_file_size;
816 r.size_is_capped = true;
817 return r;
818 }
819 r.ok = true;
820 r.size = total;
821 return r;
822}

References detect_riff(), and le32().

Referenced by build_formats(), and detect_riff().

◆ detect_sqlite()

DetectResult carving::anonymous_namespace{carving.cpp}::detect_sqlite ( const unsigned char * window,
size_t window_len,
uint64_t ,
uint64_t max_file_size,
PeekMore &  )

Definition at line 859 of file carving.cpp.

862 {
863 DetectResult r;
864 if (window_len < 100) return r;
865 // Page size at offset 16: BE16. Either a power of 2 in [512, 32768]
866 // or 1 (= 65536, special encoding).
867 const uint16_t ps_field = (static_cast<uint16_t>(window[16]) << 8) | window[17];
868 uint32_t page_size = 0;
869 if (ps_field == 1) {
870 page_size = 65536;
871 } else {
872 page_size = ps_field;
873 if (page_size < 512 || page_size > 32768) return r;
874 if ((page_size & (page_size - 1)) != 0) return r; // not pow2
875 }
876 // Page count at offset 28 (BE32)
877 const uint32_t page_count =
878 (static_cast<uint32_t>(window[28]) << 24) |
879 (static_cast<uint32_t>(window[29]) << 16) |
880 (static_cast<uint32_t>(window[30]) << 8) |
881 static_cast<uint32_t>(window[31]);
882 if (page_count == 0) return r;
883 const uint64_t total = static_cast<uint64_t>(page_size) * page_count;
884 if (total > max_file_size) {
885 r.ok = true;
886 r.size = max_file_size;
887 r.size_is_capped = true;
888 return r;
889 }
890 r.ok = true;
891 r.size = total;
892 return r;
893}

References detect_sqlite().

Referenced by build_formats(), and detect_sqlite().

◆ detect_tiff()

DetectResult carving::anonymous_namespace{carving.cpp}::detect_tiff ( const unsigned char * window,
size_t window_len,
uint64_t ,
uint64_t max_file_size,
PeekMore &  )

Definition at line 745 of file carving.cpp.

748 {
749 DetectResult r;
750 if (window_len < 8) return r;
751 // The 4-byte IFD offset must be plausible (>= 8 for a real TIFF)
752 const bool little_endian = (window[0] == 'I');
753 const uint32_t ifd_off = little_endian
754 ? le32(window + 4)
755 : be32(window + 4);
756 if (ifd_off < 8 || ifd_off > 100ull * 1024 * 1024) return r;
757 constexpr uint64_t TIFF_HARD_CAP = 50ull * 1024 * 1024;
758 r.ok = true;
759 r.size = std::min<uint64_t>(TIFF_HARD_CAP, max_file_size);
760 r.size_is_capped = true;
761 return r;
762}

References be32(), detect_tiff(), and le32().

Referenced by build_formats(), and detect_tiff().

◆ detect_zip()

DetectResult carving::anonymous_namespace{carving.cpp}::detect_zip ( const unsigned char * window,
size_t window_len,
uint64_t file_disk_off,
uint64_t max_file_size,
PeekMore & peek_more )

Definition at line 397 of file carving.cpp.

400 {
401 DetectResult r;
402 if (window_len < 30) return r;
403 // Local file header: PK\x03\x04 + version (2) + flags (2) + method (2) +
404 // mod_time (2) + mod_date (2) + crc (4) + comp_size (4) + uncomp_size (4)
405 // + name_length (2) + extra_length (2). Sanity: method must be 0..15
406 // (most commonly 0, 8). Higher values are rare and suspicious.
407 const uint16_t method = le16(window + 8);
408 if (method > 99) return r; // implausible
409
410 // Scan for EOCD: "PK\x05\x06"
411 constexpr unsigned char eocd[4] = {0x50, 0x4B, 0x05, 0x06};
412 constexpr size_t SCAN_CHUNK = 1024 * 1024;
413 std::vector<unsigned char> chunk(SCAN_CHUNK);
414
415 uint64_t last_eocd_off = 0; // file-relative
416 bool found = false;
417
418 for (size_t i = 0; i + 4 <= window_len; ++i) {
419 if (std::memcmp(window + i, eocd, 4) == 0) {
420 last_eocd_off = i;
421 found = true;
422 }
423 }
424 uint64_t total = window_len;
425 uint64_t cursor = file_disk_off + window_len;
426 unsigned char carry[3] = {0, 0, 0};
427 size_t carry_len = std::min<size_t>(3, window_len);
428 if (window_len > 0) {
429 std::memcpy(carry, window + window_len - carry_len, carry_len);
430 }
431 while (total < max_file_size) {
432 size_t got = peek_more(cursor, chunk.data(), SCAN_CHUNK);
433 if (got == 0) break;
434 std::vector<unsigned char> bnd(carry_len + std::min<size_t>(got, 3));
435 std::memcpy(bnd.data(), carry, carry_len);
436 std::memcpy(bnd.data() + carry_len, chunk.data(),
437 std::min<size_t>(got, 3));
438 for (size_t i = 0; i + 4 <= bnd.size(); ++i) {
439 if (std::memcmp(bnd.data() + i, eocd, 4) == 0) {
440 last_eocd_off = total - carry_len + i;
441 found = true;
442 }
443 }
444 for (size_t i = 0; i + 4 <= got; ++i) {
445 if (std::memcmp(chunk.data() + i, eocd, 4) == 0) {
446 last_eocd_off = total + i;
447 found = true;
448 }
449 }
450 carry_len = std::min<size_t>(3, got);
451 std::memcpy(carry, chunk.data() + got - carry_len, carry_len);
452 total += got;
453 cursor += got;
454 }
455
456 if (!found) return r;
457
458 // Read the EOCD to determine the comment length, then file size.
459 // EOCD layout:
460 // 0 PK\x05\x06
461 // 4 __le16 disk_num
462 // 6 __le16 disk_with_cd
463 // 8 __le16 entries_on_this_disk
464 // 10 __le16 entries_total
465 // 12 __le32 cd_size
466 // 16 __le32 cd_offset
467 // 20 __le16 comment_length
468 // 22 comment[comment_length]
469 unsigned char eocd_hdr[22];
470 if (peek_more(file_disk_off + last_eocd_off, eocd_hdr, 22) != 22) return r;
471 const uint16_t comment_len = le16(eocd_hdr + 20);
472 const uint64_t total_size = last_eocd_off + 22 + comment_len;
473 if (total_size > max_file_size) {
474 r.ok = true;
475 r.size = max_file_size;
476 r.size_is_capped = true;
477 return r;
478 }
479 r.ok = true;
480 r.size = total_size;
481 return r;
482}

References detect_zip(), and le16().

Referenced by build_formats(), and detect_zip().

◆ find_pattern()

size_t carving::anonymous_namespace{carving.cpp}::find_pattern ( const unsigned char * buf,
size_t buf_len,
const unsigned char * pat,
size_t pat_len,
size_t start = 0 )

Definition at line 55 of file carving.cpp.

57 {
58 if (pat_len == 0 || pat_len > buf_len) return SIZE_MAX;
59 for (size_t i = start; i + pat_len <= buf_len; ++i) {
60 if (std::memcmp(buf + i, pat, pat_len) == 0) return i;
61 }
62 return SIZE_MAX;
63}

References find_pattern().

Referenced by detect_jpeg(), and find_pattern().

◆ le16()

uint16_t carving::anonymous_namespace{carving.cpp}::le16 ( const unsigned char * p)
inline

Definition at line 26 of file carving.cpp.

26 {
27 return static_cast<uint16_t>(p[0]) |
28 (static_cast<uint16_t>(p[1]) << 8);
29}

References le16().

Referenced by detect_ole(), detect_zip(), and le16().

◆ le32()

uint32_t carving::anonymous_namespace{carving.cpp}::le32 ( const unsigned char * p)
inline

Definition at line 30 of file carving.cpp.

30 {
31 return static_cast<uint32_t>(p[0]) |
32 (static_cast<uint32_t>(p[1]) << 8) |
33 (static_cast<uint32_t>(p[2]) << 16) |
34 (static_cast<uint32_t>(p[3]) << 24);
35}

References le32().

Referenced by detect_bmp(), detect_ole(), detect_riff(), detect_tiff(), and le32().