WebSVN – QNX 8.QNX8 IFS tool – Diff – //ifstool.c

 #include <dirent.h>
 #include <utime.h>
 #endif // _MSC_VER
 // own includes
+#include "ucl/ucl.h"
+#include "minilzo.h"
 #include "buffer.h"
 #include "sha512.h"
 #include "elffile.h"
 #include "ifsfile.h"
 #include "utility.h"
 static uint32_t image_maxsize = UINT32_MAX; // default image max size (no limit)
 static uint32_t image_totalsize = 0; // image total size, measured once all the blocks have been written to the output IFS file
 static uint32_t image_align = 4; // default image alignment, as per QNX docs
 static uint32_t image_kernel_ino = 0;
 static uint32_t image_bootscript_ino = 0;
+static int startup_header_compression_flag = STARTUP_HDR_FLAGS1_COMPRESS_NONE;
 #if defined(__x86_64__)
 static char image_processor[16] = "x86_64"; // default CPU type for which this image is built, either "x86_64" or "aarch64le" (will be used to find out the right include paths under $QNX_TARGET)
 static char image_processor_base[16] = "x86_64"; // default base CPU type for which this image is built, either "x86_64" or "aarch64le" (will be used to find out the right include paths under $QNX_TARGET)
 #elif defined(__aarch64__)
 static char image_processor[16] = "aarch64le"; // default CPU type for which this image is built, either "x86_64" or "aarch64le" (will be used to find out the right include paths under $QNX_TARGET)
 static int fsentry_compare_pathnames_cb (const void *a, const void *b); // qsort() comparison callback that sorts filesystem entries by pathnames
 static void parse_line (FILE *buildfile_fp, char *line_buffer, fsentry_t **fsentries, size_t *fsentry_count, parms_t *default_parms); // parses a line in the build file and make the relevant changes to the fsentries array
 // imported function prototypes
-extern int dump_ifs_info (const char *ifs_pathname, bool want_everything); // [implemented in ifsdump.c] dumps detailed info about a particular IFS file on the standard output, returns 0 on success and >0 on error
+extern int dump_ifs_info (const char *ifs_pathname, bool want_everything, bool hide_filename); // [implemented in ifsdump.c] dumps detailed info about a particular IFS file on the standard output, returns 0 on success and >0 on error
 extern int dump_ifs_contents (const char *ifs_pathname, const char *outdir); // [implemented in ifsdump.c] dumps the IFS filesystem contents in outdir, returns 0 on success and >0 on error
 extern int dump_file_hex (const char *pathname); // [implemented in ifsdump.c] dumps the contents of pathname to stdout in mixed hexadecimal + ASCII (hex editor) format
 int32_t update_checksum (const void *data, const size_t data_len, const bool is_foreign_endianness)
+               }
                utc_time.tm_mon--; // convert month from [1-12] to [0-11]
                entry_parms.mtime = (uint32_t) mktime (&utc_time);
+            }
+         }
+         else if (strncmp (token, "compress=", 9) == 0) { REACH_TOKEN_VALUE (); startup_header_compression_flag = (strcmp (value, "1") == 0 ? STARTUP_HDR_FLAGS1_COMPRESS_ZLIB : (strcmp (value, "2") == 0 ? STARTUP_HDR_FLAGS1_COMPRESS_LZO : STARTUP_HDR_FLAGS1_COMPRESS_UCL)); }
+         else if (strcmp (token, "+compress")   == 0) startup_header_compression_flag = STARTUP_HDR_FLAGS1_COMPRESS_UCL;
+         else if (strcmp (token, "-compress")   == 0) startup_header_compression_flag = STARTUP_HDR_FLAGS1_COMPRESS_NONE;
          else if (strcmp (token, "+script")     == 0) entry_parms.is_compiled_bootscript         = true;
          else if (strcmp (token, "-script")     == 0) entry_parms.is_compiled_bootscript         = false;
          else if (strcmp (token, "+followlink") == 0) entry_parms.should_follow_symlinks         = true;
          else if (strcmp (token, "-followlink") == 0) entry_parms.should_follow_symlinks         = false;
          else if (strcmp (token, "+autolink")   == 0) entry_parms.should_autosymlink_dylib       = true;
    char path_in_ifs[MAXPATHLEN] = "";
    const char *ifs_pathname = NULL;
    const char *rootdir_pathname = NULL;
    const fsentry_t *fsentry;
    void *reallocated_ptr;
+   buffer_t compressed_imagefs;
+   uint8_t *compressor_out;
+   uint8_t *compressor_in;
+   size_t compressor_outlen;
+   size_t compressor_inlen;
    size_t reallocated_size;
    size_t available_space;
    size_t fsentry_index;
    size_t largest_index;
    size_t largest_size;
    size_t imgdir_size;
    size_t curr_offset;
+   size_t remaining_len;
    ifs_t ifs = { 0 };
    int32_t checksum;
    char *first_pathname = NULL;
    char *second_pathname = NULL;
    char *third_pathname = NULL;
    bool want_everything = false;
    bool want_help = false;
    bool want_dump = false;
    bool want_strip = false;
    bool want_hexdump = false;
+   bool hide_filename = false;
    bool is_foreign_endianness;
+   int compressor_ret;
    FILE *buildfile_fp;
    // initialize stuff
    saved_ELF_sections = (char **) malloc (4 * sizeof (char *));
    ASSERT_WITH_ERRNO (saved_ELF_sections);
          want_hexdump = true;
       else if (strcmp (argv[arg_index], "--strip") == 0)
          want_strip = true;
       else if (strcmp (argv[arg_index], "--everything") == 0)
          want_everything = true;
+      else if (strcmp (argv[arg_index], "--hide-filename") == 0)
+         hide_filename = true;
       else if (strncmp (argv[arg_index], "-v", 2) == 0) // -v[....]
          verbose_level += (int) strlen (argv[arg_index] + 1); // increase verbosity by the number of characters in this flag
       else if ((strcmp (argv[arg_index], "-l") == 0) && (arg_index + 1 < argc))
          arg_index++; // these args will be parsed once the build file is open
       else if ((strcmp (argv[arg_index], "-r") == 0) && (arg_index + 1 < argc))
       fprintf (out, "ifstool - QNX in-kernel filesystem creation utility by Pierre-Marie Baty <pm@pmbaty.com>\n");
       fprintf (out, "          version " VERSION_FMT_YYYYMMDD "\n", VERSION_ARG_YYYYMMDD);
       if (!want_help)
          fprintf (out, "error: missing parameters\n");
       fprintf (out, "usage:\n");
-      fprintf (out, "    ifstool --info [--everything] <ifs file>\n");
+      fprintf (out, "    ifstool --info [--everything] [--hide-filename] <ifs file>\n");
       fprintf (out, "    ifstool --dump [--outdir <path>] <ifs file>\n");
       fprintf (out, "    ifstool --strip [--outfile <pathname>] <ELF file>\n");
       fprintf (out, "    ifstool [-?|--help]\n");
       // mkifs [-?] [-l inputline] [-n[n]] [-o directory] [-p patchfile] [-r rootdir] [-s section] [-v] [buildfile] [directory] [outputfile]
       fprintf (out, "    ifstool [--bootfile <pathname>] [--startupfile <pathname>@<EP_from_imgbase>] [--kerneloffs <fileoffs>] [-a suffix] [-l inputline] [-n[n]] [-r rootdir] [-s section] [-v[...]] [buildfile] [directory] [outputfile]\n");
       exit (want_help ? 0 : 1);
+   }
    // else do we want info about a particular IFS ? if so, dissecate it
    else if (want_info)
-      exit (dump_ifs_info (first_pathname, want_everything));
+      exit (dump_ifs_info (first_pathname, want_everything, hide_filename));
    // else do we want to dump its contents ? if so, do so
    else if (want_dump)
       exit (dump_ifs_contents (first_pathname, (second_pathname != NULL ? second_pathname : ".")));
       ifs.offsets.startupheader = ifs.data.size; // save startup header offset for future use
       memset (&startup_header, 0, sizeof (startup_header)); // prepare startup header
       memcpy (startup_header.signature, "\xeb\x7e\xff\x00", 4); // startup header signature, i.e. 0xff7eeb
       startup_header.version       = 1;
-      startup_header.flags1        = STARTUP_HDR_FLAGS1_VIRTUAL | STARTUP_HDR_FLAGS1_TRAILER_V2; // flags, 0x21 (STARTUP_HDR_FLAGS1_VIRTUAL | STARTUP_HDR_FLAGS1_TRAILER_V2)
+      startup_header.flags1        = STARTUP_HDR_FLAGS1_VIRTUAL | STARTUP_HDR_FLAGS1_TRAILER_V2 | startup_header_compression_flag; // flags, 0x21 (STARTUP_HDR_FLAGS1_VIRTUAL | STARTUP_HDR_FLAGS1_TRAILER_V2)
       startup_header.header_size   = sizeof (startup_header); // 256
       if (strcmp (image_processor, "x86_64") == 0)
          startup_header.machine = ELF_MACHINE_X86_64; // EM_X86_64
       else if (strcmp (image_processor, "aarch64le") == 0)
          startup_header.machine = ELF_MACHINE_AARCH64; // EM_AARCH64
       startup_header.startup_vaddr = image_base + (uint32_t) startupfile_ep_from_imagebase; // [I ] Virtual Address to transfer to after IPL is done, here 0x01403008 (appears in "Entry" column for "startup.*")
       startup_header.image_paddr   = image_base + (uint32_t) bootfile_size;                 // F[IS] Physical address of image, here 0x01400f30 (appears in "Offset" column for "startup-header" which is the first entry/start of file)
       startup_header.ram_paddr     = startup_header.image_paddr;                            // [IS] Physical address of RAM to copy image to (startup_size bytes copied), here 0x01400f30 (same as above)
       startup_header.ram_size      = WILL_BE_FILLED_LATER;                                  // [ S] Amount of RAM used by the startup program and executables contained in the file system, here 0x00cd6128 i.e. 13 459 752 dec. which is 13 Mb. i.e. IFS file size minus 0x9eee (40686)
       startup_header.startup_size  = WILL_BE_FILLED_LATER;                                  // [I ] Size of startup (never compressed), here 0x02f148 or 192 840 bytes
-      startup_header.stored_size   = WILL_BE_FILLED_LATER;                                  // [I ] Size of entire image, here 0x00cd6128 (same as ram_size)
+      startup_header.stored_size   = WILL_BE_FILLED_LATER;                                  // [I ] Size of entire image file (startup + *optionally compressed* imagefs) without optional boot prefix, here 0x00cd6128
       startup_header.imagefs_size  = WILL_BE_FILLED_LATER;                                  // [ S] Size of uncompressed imagefs, here 0x00ca6fe0 or 13 266 912 bytes
       startup_header.preboot_size  = (uint16_t) bootfile_size;                              // [I ] Size of loaded before header, here 0xf30 or 3888 bytes (size of "bios.boot" file))
       ASSERT_WITH_ERRNO (Buffer_Append (&ifs.data, &startup_header, sizeof (startup_header))); // write startup header
       ASSERT_WITH_ERRNO (Buffer_PadWithZeroesTo (&ifs.data, ROUND_TO_UPPER_MULTIPLE (ifs.data.size, image_align))); // pad as necessary
    imgdir_size = ifs.data.size - ifs.offsets.imagedir; // measure image dir size and save it for future use
    // is it a bootable image with a startup file ?
    if (startupfile_pathname != NULL)
+   {
-      // start by writing the startup script data blob, if we have one
-      for (fsentry_index = 1; fsentry_index < fsentry_count; fsentry_index++)
-         if (fsentries[fsentry_index].header.ino == image_bootscript_ino)
-            break; // locate the startup script directory entry
-      if (fsentry_index < fsentry_count) // found it ?
-      {
-         if (ifs.data.size + fsentries[fsentry_index].u.file.size >= kernelfile_offset)
-            DIE_WITH_EXITCODE (1, "the compiled startup script is too big (%zd bytes, max is %zd) to fit at current offset %zd. Use --kerneloffs <addr> to relocate (warning: untested)", (size_t) fsentries[fsentry_index].u.file.size, kernelfile_offset - ifs.data.size, ifs.data.size);
-         fsentries[fsentry_index].u.file.offset = (uint32_t) (ifs.data.size - ifs.offsets.imageheader); // save file data blob offset in file structure
-         Buffer_AppendIFSFileData (&ifs.data, &fsentries[fsentry_index]); // write file data
-         fsentries[fsentry_index].UNSAVED_was_data_written = true; // and remember this file's data was written
-      }
-      // now write the filesystem entries that may fit before the kernel
+      // write the filesystem entries that may fit before the kernel
       for (;;)
+      {
          available_space = kernelfile_offset - ifs.data.size; // measure the available space until the kernel
          // look for the biggest one that can fit
       Buffer_WriteIFSDirectoryEntryAt (&ifs.data, curr_offset, &fsentries[fsentry_index]); // rewrite each dirent
       curr_offset += fsentries[fsentry_index].header.size; // advance to the next one
+   }
    // ALL CHECKSUMS AT THE VERY END
+   // compute SHA-512 checksum and V1 checksum of image block
+   if (   ( (image_header.flags & IMAGE_FLAGS_BIGENDIAN) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
+       || (!(image_header.flags & IMAGE_FLAGS_BIGENDIAN) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__   )))
+      is_foreign_endianness = true; // if the header is big endian and we're on a little endian machine, or the other way around, it's a foreign endianness
+   else
+      is_foreign_endianness = false; // else this header is for the same endianness as us
+   if (image_header.flags & IMAGE_FLAGS_TRAILER_V2) // is it a V2 trailer ?
+   {
+      SHA512 (&ifs.data.bytes[ifs.offsets.imageheader], ifs.offsets.imagetrailer - ifs.offsets.imageheader, &ifs.data.bytes[ifs.offsets.imagetrailer]); // compute SHA512 checksum and write it in place
+      checksum = update_checksum (&ifs.data.bytes[ifs.offsets.imageheader], ifs.offsets.imagetrailer + SHA512_DIGEST_LENGTH - ifs.offsets.imageheader, is_foreign_endianness); // compute old checksum
+      memcpy (&ifs.data.bytes[ifs.offsets.imagetrailer + SHA512_DIGEST_LENGTH], &checksum, 4); // and write it in place
+   }
+   else // old V1 trailer
+   {
+      checksum = update_checksum (&ifs.data.bytes[ifs.offsets.imageheader], ifs.offsets.imagetrailer - ifs.offsets.imageheader, is_foreign_endianness); // compute old checksum
+      memcpy (&ifs.data.bytes[ifs.offsets.imagetrailer], &checksum, 4); // and write it in place
+   }
+   // should we compress the image block ?
+   if (startup_header_compression_flag != STARTUP_HDR_FLAGS1_COMPRESS_NONE)
+   {
+      // it appears mkifs compresses data in blocks, prefixed by 2-byte block size in BIG ENDIAN
+      Buffer_InitWithSize (&compressed_imagefs, image_header.image_size * 11 / 10); // mallocate and add 10% for safety
+      compressed_imagefs.size = 0;
+      compressor_in = &ifs.data.bytes[ifs.offsets.imageheader]; // point at the start of the data to compress
+      compressor_out = &compressed_imagefs.bytes[2]; // point after the compressed block size word
+      remaining_len = ifs.data.size - ifs.offsets.imageheader; // see how many bytes there are to compress
+      if (startup_header_compression_flag == STARTUP_HDR_FLAGS1_COMPRESS_UCL)
+         ASSERT (ucl_init () == UCL_E_OK, "UCL library initialization failed -- please recompile this tool with less aggressive optimizations");
+      else if (startup_header_compression_flag == STARTUP_HDR_FLAGS1_COMPRESS_LZO)
+         ASSERT (ucl_init () == UCL_E_OK, "LZO library initialization failed -- please recompile this tool with less aggressive optimizations");
+      else if (startup_header_compression_flag == STARTUP_HDR_FLAGS1_COMPRESS_ZLIB)
+         DIE_WITH_EXITCODE (1, "unimplemented compression scheme: zlib (FIXME)");
+      else
+         DIE_WITH_EXITCODE (1, "unsupported compression flags: 0x%2x", startup_header_compression_flag);
+      // run the compressible payload (the imagefs) through the right compression algorithm
+      while (remaining_len > 0)
+      {
+         compressor_inlen = (ucl_uint) remaining_len; // size the compressor input appropriately
+         if (compressor_inlen > 65536)
+            compressor_inlen = 65536; // cap it to a VERY conservative value
+         if (startup_header_compression_flag == STARTUP_HDR_FLAGS1_COMPRESS_UCL)
+         {
+            // UCL compression. NOTE: the decompressor function used in startup-x86 is "ucl_nrv2b_decompress_8". Also compression level is hardcoded at 9 in mkifs, but can range from 1 to 10
+            static ucl_uint ucl_outlen; // have a different variable because of pointer size mismatch
+            while (((compressor_ret = ucl_nrv2b_99_compress (compressor_in, (ucl_uint) compressor_inlen, compressor_out, &ucl_outlen, NULL, 9, NULL, NULL)) == UCL_E_OK) && (ucl_outlen > 0xFFFF))
+               compressor_inlen -= 4096; // as long as we can't produce compressed blocks whose size can fit in 2 bytes, try again with 4kb input less
+            ASSERT (compressor_ret == UCL_E_OK, "UCL compression error: ucl_nrv2b_99_compress() returned %d", compressor_ret); // make sure it's not a compression error
+            compressor_outlen = ucl_outlen; // cast back to size_t
+         }
+         else if (startup_header_compression_flag == STARTUP_HDR_FLAGS1_COMPRESS_LZO)
+         {
+            // LZO compression. NOTE: the compressor function used in mkifs is "lzo1x_999_compress" which is from the full LZO package... I use use minilzo, but I have to admit the compression ratio of the full LZO is *much* better.
+            static lzo_align_t lzo_workmem[(LZO1X_1_MEM_COMPRESS + (sizeof (lzo_align_t) - 1)) / sizeof(lzo_align_t)]; // heap-allocated aligned buffer
+            static lzo_uint lzo_outlen; // have a different variable because of pointer size mismatch
+            while (((compressor_ret = lzo1x_1_compress (compressor_in, compressor_inlen, compressor_out, &lzo_outlen, lzo_workmem)) == UCL_E_OK) && (lzo_outlen > 0xFFFF))
+               compressor_inlen -= 4096; // as long as we can't produce compressed blocks whose size can fit in 2 bytes, try again with 4kb input less
+            ASSERT (compressor_ret == LZO_E_OK, "LZO compression error: lzo1x_1_compress() returned %d", compressor_ret); // make sure it's not a compression error
+            compressor_outlen = lzo_outlen; // cast back to size_t
+         }
+         else if (startup_header_compression_flag == STARTUP_HDR_FLAGS1_COMPRESS_ZLIB)
+         {
+            // Zlib (TODO)
+         }
+         // the compression produced a block smaller than 65536 bytes
+         LOG_DEBUG ("compressed block size %zd", compressor_outlen);
+         compressed_imagefs.bytes[compressed_imagefs.size + 0] = (uint8_t) (compressor_outlen >> 8); // write compressed block size word (in big endian)
+         compressed_imagefs.bytes[compressed_imagefs.size + 1] = (uint8_t) (compressor_outlen >> 0);
+         compressed_imagefs.size += 2 + (size_t) compressor_outlen; // advance in compression buffer by the compressed block size word plus the compressed block size
+         remaining_len -= compressor_inlen; // see how many bytes remain to compress
+         compressor_in += compressor_inlen; // advance in input stream
+         compressor_out += 2 + compressor_outlen; // advance in output stream
+      }
+      compressed_imagefs.bytes[compressed_imagefs.size + 0] = 0; // write the end of stream marker (empty block with nil size)
+      compressed_imagefs.bytes[compressed_imagefs.size + 1] = 0;
+      compressed_imagefs.size += 2;
+      LOG_INFO ("compressed %zd bytes into %zd bytes\n", ifs.data.size - ifs.offsets.imageheader, compressed_imagefs.size);
+      /////////////////////////////////////////////////////////////
+      // WARNING: ALL IFS OFFSETS BECOME INVALID PAST THIS POINT //
+      /////////////////////////////////////////////////////////////
+      // now place the compressed buffer in the payload at the imagefs offset
+      ASSERT_WITH_ERRNO (Buffer_WriteBufferAt (&ifs.data, ifs.offsets.imageheader, &compressed_imagefs));
+      ifs.data.size = ifs.offsets.imageheader + compressed_imagefs.size; // update IFS data size
+      Buffer_Forget (&compressed_imagefs);
+      // fix the stored size in the startup header
+      startup_header.stored_size = (uint32_t) (ifs.data.size - ifs.offsets.startupheader);
+      ASSERT_WITH_ERRNO (Buffer_WriteAt (&ifs.data, ifs.offsets.startupheader, &startup_header, sizeof (startup_header))); // write the final startup header at its right offset
+   }
    // do we have a startup file ? if so, this is a bootable image
    if (startupfile_pathname != NULL)
+   {
       // compute SHA-512 checksum and V1 checksum of startup block
       if (   ( (startup_header.flags1 & STARTUP_HDR_FLAGS1_BIGENDIAN) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
-          || (!(startup_header.flags1 & STARTUP_HDR_FLAGS1_BIGENDIAN) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)))
+          || (!(startup_header.flags1 & STARTUP_HDR_FLAGS1_BIGENDIAN) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__   )))
          is_foreign_endianness = true; // if the header is big endian and we're on a little endian machine, or the other way around, it's a foreign endianness
       else
          is_foreign_endianness = false; // else this header is for the same endianness as us
       if (startup_header.flags1 & STARTUP_HDR_FLAGS1_TRAILER_V2) // is it a V2 trailer ?
       else // old V1 trailer
+      {
          checksum = update_checksum (&ifs.data.bytes[ifs.offsets.startupheader], ifs.offsets.startuptrailer - ifs.offsets.startupheader, is_foreign_endianness); // compute old checksum
          memcpy (&ifs.data.bytes[ifs.offsets.startuptrailer], &checksum, 4); // and write it in place
+      }
-   }
-   // compute SHA-512 checksum and V1 checksum of image block
-   if (   ( (image_header.flags & IMAGE_FLAGS_BIGENDIAN) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
-       || (!(image_header.flags & IMAGE_FLAGS_BIGENDIAN) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)))
-      is_foreign_endianness = true; // if the header is big endian and we're on a little endian machine, or the other way around, it's a foreign endianness
-   else
-      is_foreign_endianness = false; // else this header is for the same endianness as us
-   if (image_header.flags & IMAGE_FLAGS_TRAILER_V2) // is it a V2 trailer ?
-   {
-      SHA512 (&ifs.data.bytes[ifs.offsets.imageheader], ifs.offsets.imagetrailer - ifs.offsets.imageheader, &ifs.data.bytes[ifs.offsets.imagetrailer]); // compute SHA512 checksum and write it in place
-      checksum = update_checksum (&ifs.data.bytes[ifs.offsets.imageheader], ifs.offsets.imagetrailer + SHA512_DIGEST_LENGTH - ifs.offsets.imageheader, is_foreign_endianness); // compute old checksum
-      memcpy (&ifs.data.bytes[ifs.offsets.imagetrailer + SHA512_DIGEST_LENGTH], &checksum, 4); // and write it in place
-   }
-   else // old V1 trailer
-   {
-      checksum = update_checksum (&ifs.data.bytes[ifs.offsets.imageheader], ifs.offsets.imagetrailer - ifs.offsets.imageheader, is_foreign_endianness); // compute old checksum
-      memcpy (&ifs.data.bytes[ifs.offsets.imagetrailer], &checksum, 4); // and write it in place
+   }
    // now rewrite IFS with the correct checksums
    ASSERT_WITH_ERRNO (Buffer_WriteToFile (&ifs.data, (ifs_pathname != NULL ? ifs_pathname : "<stdout>")));

Subversion Repositories QNX 8.QNX8 IFS tool

QNX 8.QNX8 IFS tool//ifstool.c – Rev 25 → 26