/* * Create a squashfs filesystem. This is a highly compressed read only * filesystem. * * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, * 2012, 2013, 2014, 2017, 2019, 2021, 2022, 2023 * Phillip Lougher * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2, * or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * * mksquashfs.c */ #define FALSE 0 #define TRUE 1 #define MAX_LINE 16384 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef __linux__ #include #include #include #else #include #endif #include "squashfs_fs.h" #include "squashfs_swap.h" #include "mksquashfs.h" #include "sort.h" #include "pseudo.h" #include "compressor.h" #include "xattr.h" #include "action.h" #include "mksquashfs_error.h" #include "progressbar.h" #include "info.h" #include "caches-queues-lists.h" #include "read_fs.h" #include "restore.h" #include "process_fragments.h" #include "fnmatch_compat.h" #include "tar.h" #include "merge_sort.h" /* Compression options */ int noF = FALSE; int noI = FALSE; int noId = FALSE; int noD = FALSE; int noX = FALSE; /* block size used to build filesystem */ int block_size = SQUASHFS_FILE_SIZE; int block_log; /* Fragment options, are fragments in filesystem and are they used for tailends? */ int no_fragments = FALSE; int always_use_fragments = FALSE; /* Are duplicates detected in fileystem ? */ int duplicate_checking = TRUE; /* Are filesystems exportable via NFS? */ int exportable = TRUE; /* Are sparse files detected and stored? */ int sparse_files = TRUE; /* Options which override root inode settings */ int root_mode_opt = FALSE; mode_t root_mode; int root_uid_opt = FALSE; unsigned int root_uid; int root_gid_opt = FALSE; unsigned int root_gid; unsigned int root_time; int root_time_opt = FALSE; /* Values that override uids and gids for all files and directories */ int global_uid_opt = FALSE; unsigned int global_uid; int global_gid_opt = FALSE; unsigned int global_gid; /* Do pseudo uids and guids override -all-root, -force-uid and -force-gid? */ int pseudo_override = FALSE; /* Time value over-ride options */ unsigned int mkfs_time; int mkfs_time_opt = FALSE; unsigned int all_time; int all_time_opt = FALSE; int clamping = TRUE; /* Is max depth option in effect, and max depth to descend into directories */ int max_depth_opt = FALSE; unsigned int max_depth; /* how should Mksquashfs treat the source files? */ int tarstyle = FALSE; int keep_as_directory = FALSE; /* should Mksquashfs read files from stdin, like cpio? */ int cpiostyle = FALSE; char filename_terminator = '\n'; /* Should Mksquashfs detect hardlinked files? */ int no_hardlinks = FALSE; /* Should Mksquashfs cross filesystem boundaries? */ int one_file_system = FALSE; int one_file_system_x = FALSE; dev_t *source_dev; dev_t cur_dev; /* Is Mksquashfs processing a tarfile? */ int tarfile = FALSE; /* Is Mksquashfs reading a pseudo file from stdin? */ int pseudo_stdin = FALSE; /* Is Mksquashfs storing Xattrs, or excluding/including xattrs using regexs? */ int no_xattrs = XATTR_DEF; unsigned int xattr_bytes = 0, total_xattr_bytes = 0; regex_t *xattr_exclude_preg = NULL; regex_t *xattr_include_preg = NULL; /* Does Mksquashfs print a summary and other information when running? */ int quiet = FALSE; /* Does Mksquashfs display filenames as they are archived? */ int silent = TRUE; /* Is Mksquashfs using the older non-wildcard exclude code? */ int old_exclude = TRUE; /* Is Mksquashfs using regexs in exclude file matching (default wildcards)? */ int use_regex = FALSE; /* Will Mksquashfs pad the filesystem to a multiple of 4 Kbytes? */ int nopad = FALSE; /* Should Mksquashfs treat normally ignored errors as fatal? */ int exit_on_error = FALSE; /* Is filesystem stored at an offset from the start of the block device/file? */ long long start_offset = 0; /* File count statistics used to print summary and fill in superblock */ unsigned int file_count = 0, sym_count = 0, dev_count = 0, dir_count = 0, fifo_count = 0, sock_count = 0, id_count = 0; long long hardlnk_count = 0; /* superblock attributes */ struct squashfs_super_block sBlk; /* write position within data section */ long long bytes = 0, total_bytes = 0; /* in memory directory table - possibly compressed */ char *directory_table = NULL; long long directory_bytes = 0, directory_size = 0, total_directory_bytes = 0; /* cached directory table */ char *directory_data_cache = NULL; unsigned int directory_cache_bytes = 0, directory_cache_size = 0; /* in memory inode table - possibly compressed */ char *inode_table = NULL; long long inode_bytes = 0, inode_size = 0, total_inode_bytes = 0; /* cached inode table */ char *data_cache = NULL; unsigned int cache_bytes = 0, cache_size = 0, inode_count = 0; /* inode lookup table */ squashfs_inode *inode_lookup_table = NULL; struct inode_info *inode_info[INODE_HASH_SIZE]; /* hash tables used to do fast duplicate searches in duplicate check */ struct file_info **dupl_frag; struct file_info **dupl_block; unsigned int dup_files = 0; int exclude = 0; struct exclude_info *exclude_paths = NULL; struct path_entry { char *name; regex_t *preg; struct pathname *paths; }; struct pathnames *paths = NULL; struct pathname *path = NULL; struct pathname *stickypath = NULL; unsigned int fragments = 0; struct squashfs_fragment_entry *fragment_table = NULL; int fragments_outstanding = 0; int fragments_locked = FALSE; /* current inode number for directories and non directories */ unsigned int inode_no = 1; unsigned int root_inode_number = 0; /* list of source dirs/files */ int source = 0; char **source_path; int option_offset; /* flag whether destination file is a block device */ int block_device = FALSE; /* flag indicating whether files are sorted using sort list(s) */ int sorted = FALSE; /* save destination file name for deleting on error */ char *destination_file = NULL; struct id *id_hash_table[ID_ENTRIES]; struct id *id_table[SQUASHFS_IDS], *sid_table[SQUASHFS_IDS]; unsigned int uid_count = 0, guid_count = 0; unsigned int sid_count = 0, suid_count = 0, sguid_count = 0; /* caches used to store buffers being worked on, and queues * used to send buffers between threads */ struct cache *reader_buffer, *fragment_buffer, *reserve_cache; struct cache *bwriter_buffer, *fwriter_buffer; struct queue *to_reader, *to_deflate, *to_writer, *from_writer, *to_frag, *locked_fragment, *to_process_frag; struct seq_queue *to_main; /* pthread threads and mutexes */ pthread_t reader_thread, writer_thread, main_thread; pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread; pthread_t *restore_thread = NULL; pthread_mutex_t fragment_mutex = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t pos_mutex = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t dup_mutex = PTHREAD_MUTEX_INITIALIZER; /* reproducible image queues and threads */ struct seq_queue *to_order; pthread_t order_thread; pthread_cond_t fragment_waiting = PTHREAD_COND_INITIALIZER; int sequence_count = 0; int reproducible = REP_DEF; /* user options that control parallelisation */ int processors = -1; int bwriter_size; /* Compressor options (-X) and initialised compressor (-comp XXX) */ int comp_opts = FALSE; int X_opt_parsed = FALSE; struct compressor *comp = NULL; int compressor_opt_parsed = FALSE; void *stream = NULL; /* root of the in-core directory structure */ struct dir_info *root_dir; /* log file */ FILE *log_fd; int logging=FALSE; /* file descriptor of the output filesystem */ int fd; /* Variables used for appending */ int appending = TRUE; /* restore orignal filesystem state if appending to existing filesystem is * cancelled */ char *sdata_cache, *sdirectory_data_cache, *sdirectory_compressed; long long sbytes, stotal_bytes; long long sinode_bytes, stotal_inode_bytes; long long sdirectory_bytes, stotal_directory_bytes; unsigned int scache_bytes, sdirectory_cache_bytes, sdirectory_compressed_bytes, sinode_count = 0, sfile_count, ssym_count, sdev_count, sdir_count, sfifo_count, ssock_count, sdup_files; unsigned int sfragments; /* list of root directory entries read from original filesystem */ int old_root_entries = 0; struct old_root_entry_info *old_root_entry; /* fragment to file mapping used when appending */ struct append_file **file_mapping; /* recovery file for abnormal exit on appending */ char *recovery_file = NULL; char *recovery_pathname = NULL; int recover = TRUE; /* list of options that have an argument */ char *option_table[] = { "comp", "b", "mkfs-time", "fstime", "all-time", "root-mode", "force-uid", "force-gid", "action", "log-action", "true-action", "false-action", "action-file", "log-action-file", "true-action-file", "false-action-file", "p", "pf", "sort", "root-becomes", "recover", "recovery-path", "throttle", "limit", "processors", "mem", "offset", "o", "log", "a", "va", "ta", "fa", "af", "vaf", "taf", "faf", "read-queue", "write-queue", "fragment-queue", "root-time", "root-uid", "root-gid", "xattrs-exclude", "xattrs-include", "xattrs-add", "default-mode", "default-uid", "default-gid", "mem-percent", NULL }; char *sqfstar_option_table[] = { "comp", "b", "mkfs-time", "fstime", "all-time", "root-mode", "force-uid", "force-gid", "throttle", "limit", "processors", "mem", "offset", "o", "root-time", "root-uid", "root-gid", "xattrs-exclude", "xattrs-include", "xattrs-add", "p", "pf", "default-mode", "default-uid", "default-gid", "mem-percent", NULL }; static char *read_from_disk(long long start, unsigned int avail_bytes); static void add_old_root_entry(char *name, squashfs_inode inode, unsigned int inode_number, int type); static struct file_info *duplicate(int *dup, int *block_dup, long long file_size, long long bytes, unsigned int *block_list, long long start, struct dir_ent *dir_ent, struct file_buffer *file_buffer, int blocks, long long sparse, int bl_hash); static struct dir_info *dir_scan1(char *, char *, struct pathnames *, struct dir_ent *(_readdir)(struct dir_info *), unsigned int); static void dir_scan2(struct dir_info *dir, struct pseudo *pseudo); static void dir_scan3(struct dir_info *dir); static void dir_scan4(struct dir_info *dir, int symlink); static void dir_scan5(struct dir_info *dir); static void dir_scan6(struct dir_info *dir); static void dir_scan7(squashfs_inode *inode, struct dir_info *dir_info); static struct dir_ent *scan1_readdir(struct dir_info *dir); static struct dir_ent *scan1_single_readdir(struct dir_info *dir); static struct dir_ent *scan1_encomp_readdir(struct dir_info *dir); static struct file_info *add_non_dup(long long file_size, long long bytes, unsigned int blocks, long long sparse, unsigned int *block_list, long long start, struct fragment *fragment, unsigned short checksum, unsigned short fragment_checksum, int checksum_flag, int checksum_frag_flag, int blocks_dup, int frag_dup, int bl_hash); long long generic_write_table(long long, void *, int, void *, int); void restorefs(); struct dir_info *scan1_opendir(char *pathname, char *subpath, unsigned int depth); static void write_filesystem_tables(struct squashfs_super_block *sBlk); unsigned short get_checksum_mem(char *buff, int bytes); static void check_usable_phys_mem(int total_mem); static void print_summary(); void write_destination(int fd, long long byte, long long bytes, void *buff); static int old_excluded(char *filename, struct stat *buf); void prep_exit() { if(restore_thread) { if(pthread_self() == *restore_thread) { /* * Recursive failure when trying to restore filesystem! * Nothing to do except to exit, otherwise we'll just * appear to hang. The user should be able to restore * from the recovery file (which is why it was added, in * case of catastrophic failure in Mksquashfs) */ exit(1); } else { /* signal the restore thread to restore */ pthread_kill(*restore_thread, SIGUSR1); pthread_exit(NULL); } } else if(!appending) { if(destination_file && !block_device) unlink(destination_file); } else if(recovery_file) unlink(recovery_file); } int add_overflow(int a, int b) { return (INT_MAX - a) < b; } int shift_overflow(int a, int shift) { return (INT_MAX >> shift) < a; } int multiply_overflow(int a, int multiplier) { return (INT_MAX / multiplier) < a; } int multiply_overflowll(long long a, int multiplier) { return (LLONG_MAX / multiplier) < a; } #define MKINODE(A) ((squashfs_inode)(((squashfs_inode) inode_bytes << 16) \ + (((char *)A) - data_cache))) void restorefs() { int i, res; ERROR("Exiting - restoring original filesystem!\n\n"); bytes = sbytes; memcpy(data_cache, sdata_cache, cache_bytes = scache_bytes); memcpy(directory_data_cache, sdirectory_data_cache, sdirectory_cache_bytes); directory_cache_bytes = sdirectory_cache_bytes; inode_bytes = sinode_bytes; directory_bytes = sdirectory_bytes; memcpy(directory_table + directory_bytes, sdirectory_compressed, sdirectory_compressed_bytes); directory_bytes += sdirectory_compressed_bytes; total_bytes = stotal_bytes; total_inode_bytes = stotal_inode_bytes; total_directory_bytes = stotal_directory_bytes; inode_count = sinode_count; file_count = sfile_count; sym_count = ssym_count; dev_count = sdev_count; dir_count = sdir_count; fifo_count = sfifo_count; sock_count = ssock_count; dup_files = sdup_files; fragments = sfragments; id_count = sid_count; restore_xattrs(); write_filesystem_tables(&sBlk); if(!block_device) { int res = ftruncate(fd, bytes); if(res != 0) BAD_ERROR("Failed to truncate dest file because %s\n", strerror(errno)); } if(!nopad && (i = bytes & (4096 - 1))) { char temp[4096] = {0}; write_destination(fd, bytes, 4096 - i, temp); } res = close(fd); if(res == -1) BAD_ERROR("Failed to close output filesystem, close returned %s\n", strerror(errno)); if(recovery_file) unlink(recovery_file); if(!quiet) print_summary(); exit(1); } void sighandler(int arg) { EXIT_MKSQUASHFS(); } static int mangle2(void *strm, char *d, char *s, int size, int block_size, int uncompressed, int data_block) { int error, c_byte = 0; if(!uncompressed) { c_byte = compressor_compress(comp, strm, d, s, size, block_size, &error); if(c_byte == -1) BAD_ERROR("mangle2:: %s compress failed with error " "code %d\n", comp->name, error); } if(c_byte == 0 || c_byte >= size) { memcpy(d, s, size); return size | (data_block ? SQUASHFS_COMPRESSED_BIT_BLOCK : SQUASHFS_COMPRESSED_BIT); } return c_byte; } int mangle(char *d, char *s, int size, int block_size, int uncompressed, int data_block) { return mangle2(stream, d, s, size, block_size, uncompressed, data_block); } static void *get_inode(int req_size) { int data_space; unsigned short c_byte; while(cache_bytes >= SQUASHFS_METADATA_SIZE) { if((inode_size - inode_bytes) < ((SQUASHFS_METADATA_SIZE << 1)) + 2) { void *it = realloc(inode_table, inode_size + (SQUASHFS_METADATA_SIZE << 1) + 2); if(it == NULL) MEM_ERROR(); inode_table = it; inode_size += (SQUASHFS_METADATA_SIZE << 1) + 2; } c_byte = mangle(inode_table + inode_bytes + BLOCK_OFFSET, data_cache, SQUASHFS_METADATA_SIZE, SQUASHFS_METADATA_SIZE, noI, 0); TRACE("Inode block @ 0x%x, size %d\n", inode_bytes, c_byte); SQUASHFS_SWAP_SHORTS(&c_byte, inode_table + inode_bytes, 1); inode_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + BLOCK_OFFSET; total_inode_bytes += SQUASHFS_METADATA_SIZE + BLOCK_OFFSET; memmove(data_cache, data_cache + SQUASHFS_METADATA_SIZE, cache_bytes - SQUASHFS_METADATA_SIZE); cache_bytes -= SQUASHFS_METADATA_SIZE; } data_space = (cache_size - cache_bytes); if(data_space < req_size) { int realloc_size = cache_size == 0 ? ((req_size + SQUASHFS_METADATA_SIZE) & ~(SQUASHFS_METADATA_SIZE - 1)) : req_size - data_space; void *dc = realloc(data_cache, cache_size + realloc_size); if(dc == NULL) MEM_ERROR(); cache_size += realloc_size; data_cache = dc; } cache_bytes += req_size; return data_cache + cache_bytes - req_size; } long long read_bytes(int fd, void *buff, long long bytes) { long long res, count; for(count = 0; count < bytes; count += res) { int len = (bytes - count) > MAXIMUM_READ_SIZE ? MAXIMUM_READ_SIZE : bytes - count; res = read(fd, buff + count, len); if(res < 1) { if(res == 0) goto bytes_read; else if(errno != EINTR) { ERROR("Read failed because %s\n", strerror(errno)); return -1; } else res = 0; } } bytes_read: return count; } int read_fs_bytes(int fd, long long byte, long long bytes, void *buff) { off_t off = byte; int res = 1; TRACE("read_fs_bytes: reading from position 0x%llx, bytes %lld\n", byte, bytes); pthread_cleanup_push((void *) pthread_mutex_unlock, &pos_mutex); pthread_mutex_lock(&pos_mutex); if(lseek(fd, start_offset + off, SEEK_SET) == -1) { ERROR("read_fs_bytes: Lseek on destination failed because %s, " "offset=0x%llx\n", strerror(errno), start_offset + off); res = 0; } else if(read_bytes(fd, buff, bytes) < bytes) { ERROR("Read on destination failed\n"); res = 0; } pthread_cleanup_pop(1); return res; } int write_bytes(int fd, void *buff, long long bytes) { long long res, count; for(count = 0; count < bytes; count += res) { int len = (bytes - count) > MAXIMUM_READ_SIZE ? MAXIMUM_READ_SIZE : bytes - count; res = write(fd, buff + count, len); if(res == -1) { if(errno != EINTR) { ERROR("Write failed because %s\n", strerror(errno)); return -1; } res = 0; } } return 0; } void write_destination(int fd, long long byte, long long bytes, void *buff) { off_t off = byte; pthread_cleanup_push((void *) pthread_mutex_unlock, &pos_mutex); pthread_mutex_lock(&pos_mutex); if(lseek(fd, start_offset + off, SEEK_SET) == -1) { ERROR("write_destination: Lseek on destination " "failed because %s, offset=0x%llx\n", strerror(errno), start_offset + off); BAD_ERROR("Probably out of space on output %s\n", block_device ? "block device" : "filesystem"); } if(write_bytes(fd, buff, bytes) == -1) BAD_ERROR("Failed to write to output %s\n", block_device ? "block device" : "filesystem"); pthread_cleanup_pop(1); } static long long write_inodes() { unsigned short c_byte; int avail_bytes; char *datap = data_cache; long long start_bytes = bytes; while(cache_bytes) { if(inode_size - inode_bytes < ((SQUASHFS_METADATA_SIZE << 1) + 2)) { void *it = realloc(inode_table, inode_size + ((SQUASHFS_METADATA_SIZE << 1) + 2)); if(it == NULL) MEM_ERROR(); inode_size += (SQUASHFS_METADATA_SIZE << 1) + 2; inode_table = it; } avail_bytes = cache_bytes > SQUASHFS_METADATA_SIZE ? SQUASHFS_METADATA_SIZE : cache_bytes; c_byte = mangle(inode_table + inode_bytes + BLOCK_OFFSET, datap, avail_bytes, SQUASHFS_METADATA_SIZE, noI, 0); TRACE("Inode block @ 0x%x, size %d\n", inode_bytes, c_byte); SQUASHFS_SWAP_SHORTS(&c_byte, inode_table + inode_bytes, 1); inode_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + BLOCK_OFFSET; total_inode_bytes += avail_bytes + BLOCK_OFFSET; datap += avail_bytes; cache_bytes -= avail_bytes; } write_destination(fd, bytes, inode_bytes, inode_table); bytes += inode_bytes; return start_bytes; } static long long write_directories() { unsigned short c_byte; int avail_bytes; char *directoryp = directory_data_cache; long long start_bytes = bytes; while(directory_cache_bytes) { if(directory_size - directory_bytes < ((SQUASHFS_METADATA_SIZE << 1) + 2)) { void *dt = realloc(directory_table, directory_size + ((SQUASHFS_METADATA_SIZE << 1) + 2)); if(dt == NULL) MEM_ERROR(); directory_size += (SQUASHFS_METADATA_SIZE << 1) + 2; directory_table = dt; } avail_bytes = directory_cache_bytes > SQUASHFS_METADATA_SIZE ? SQUASHFS_METADATA_SIZE : directory_cache_bytes; c_byte = mangle(directory_table + directory_bytes + BLOCK_OFFSET, directoryp, avail_bytes, SQUASHFS_METADATA_SIZE, noI, 0); TRACE("Directory block @ 0x%x, size %d\n", directory_bytes, c_byte); SQUASHFS_SWAP_SHORTS(&c_byte, directory_table + directory_bytes, 1); directory_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + BLOCK_OFFSET; total_directory_bytes += avail_bytes + BLOCK_OFFSET; directoryp += avail_bytes; directory_cache_bytes -= avail_bytes; } write_destination(fd, bytes, directory_bytes, directory_table); bytes += directory_bytes; return start_bytes; } static long long write_id_table() { unsigned int id_bytes = SQUASHFS_ID_BYTES(id_count); unsigned int p[id_count]; int i; TRACE("write_id_table: ids %d, id_bytes %d\n", id_count, id_bytes); for(i = 0; i < id_count; i++) { TRACE("write_id_table: id index %d, id %d", i, id_table[i]->id); SQUASHFS_SWAP_INTS(&id_table[i]->id, p + i, 1); } return generic_write_table(id_bytes, p, 0, NULL, noI || noId); } static struct id *get_id(unsigned int id) { int hash = ID_HASH(id); struct id *entry = id_hash_table[hash]; for(; entry; entry = entry->next) if(entry->id == id) break; return entry; } struct id *create_id(unsigned int id) { int hash = ID_HASH(id); struct id *entry = malloc(sizeof(struct id)); if(entry == NULL) MEM_ERROR(); entry->id = id; entry->index = id_count ++; entry->flags = 0; entry->next = id_hash_table[hash]; id_hash_table[hash] = entry; id_table[entry->index] = entry; return entry; } unsigned int get_uid(unsigned int uid) { struct id *entry = get_id(uid); if(entry == NULL) { if(id_count == SQUASHFS_IDS) BAD_ERROR("Out of uids!\n"); entry = create_id(uid); } if((entry->flags & ISA_UID) == 0) { entry->flags |= ISA_UID; uid_count ++; } return entry->index; } unsigned int get_guid(unsigned int guid) { struct id *entry = get_id(guid); if(entry == NULL) { if(id_count == SQUASHFS_IDS) BAD_ERROR("Out of gids!\n"); entry = create_id(guid); } if((entry->flags & ISA_GID) == 0) { entry->flags |= ISA_GID; guid_count ++; } return entry->index; } char *pathname(struct dir_ent *dir_ent) { static char *pathname = NULL; static int size = ALLOC_SIZE; if (dir_ent->nonstandard_pathname) return dir_ent->nonstandard_pathname; if(pathname == NULL) { pathname = malloc(ALLOC_SIZE); if(pathname == NULL) MEM_ERROR(); } for(;;) { int res = snprintf(pathname, size, "%s/%s", dir_ent->our_dir->pathname, dir_ent->source_name ? : dir_ent->name); if(res < 0) BAD_ERROR("snprintf failed in pathname\n"); else if(res >= size) { /* * pathname is too small to contain the result, so * increase it and try again */ size = (res + ALLOC_SIZE) & ~(ALLOC_SIZE - 1); pathname = realloc(pathname, size); if(pathname == NULL) MEM_ERROR(); } else break; } return pathname; } char *subpathname(struct dir_ent *dir_ent) { static char *subpath = NULL; static int size = ALLOC_SIZE; int res; if(subpath == NULL) { subpath = malloc(ALLOC_SIZE); if(subpath == NULL) MEM_ERROR(); } for(;;) { if(dir_ent->our_dir->subpath[0] != '\0') res = snprintf(subpath, size, "%s/%s", dir_ent->our_dir->subpath, dir_ent->name); else res = snprintf(subpath, size, "/%s", dir_ent->name); if(res < 0) BAD_ERROR("snprintf failed in subpathname\n"); else if(res >= size) { /* * subpath is too small to contain the result, so * increase it and try again */ size = (res + ALLOC_SIZE) & ~(ALLOC_SIZE - 1); subpath = realloc(subpath, size); if(subpath == NULL) MEM_ERROR(); } else break; } return subpath; } static inline unsigned int get_inode_no(struct inode_info *inode) { return inode->inode_number; } static inline unsigned int get_parent_no(struct dir_info *dir) { return dir->depth ? get_inode_no(dir->dir_ent->inode) : inode_no; } static inline time_t get_time(time_t time) { if(all_time_opt) { if(clamping) return time > all_time ? all_time : time; else return all_time; } return time; } squashfs_inode create_inode(struct dir_info *dir_info, struct dir_ent *dir_ent, int type, long long byte_size, long long start_block, unsigned int offset, unsigned int *block_list, struct fragment *fragment, struct directory *dir_in, long long sparse) { struct stat *buf = &dir_ent->inode->buf; union squashfs_inode_header inode_header; struct squashfs_base_inode_header *base = &inode_header.base; void *inode; char *filename = pathname(dir_ent); int nlink = dir_ent->inode->nlink; int xattr = read_xattrs(dir_ent, type); unsigned int uid, gid; switch(type) { case SQUASHFS_FILE_TYPE: if(dir_ent->inode->nlink > 1 || byte_size >= (1LL << 32) || start_block >= (1LL << 32) || sparse || IS_XATTR(xattr)) type = SQUASHFS_LREG_TYPE; break; case SQUASHFS_DIR_TYPE: if(dir_info->dir_is_ldir || IS_XATTR(xattr)) type = SQUASHFS_LDIR_TYPE; break; case SQUASHFS_SYMLINK_TYPE: if(IS_XATTR(xattr)) type = SQUASHFS_LSYMLINK_TYPE; break; case SQUASHFS_BLKDEV_TYPE: if(IS_XATTR(xattr)) type = SQUASHFS_LBLKDEV_TYPE; break; case SQUASHFS_CHRDEV_TYPE: if(IS_XATTR(xattr)) type = SQUASHFS_LCHRDEV_TYPE; break; case SQUASHFS_FIFO_TYPE: if(IS_XATTR(xattr)) type = SQUASHFS_LFIFO_TYPE; break; case SQUASHFS_SOCKET_TYPE: if(IS_XATTR(xattr)) type = SQUASHFS_LSOCKET_TYPE; break; } if(!pseudo_override && global_uid_opt) uid = global_uid; else uid = buf->st_uid; if(!pseudo_override && global_gid_opt) gid = global_gid; else gid = buf->st_gid; base->mode = SQUASHFS_MODE(buf->st_mode); base->inode_type = type; base->uid = get_uid(uid); base->guid = get_guid(gid); base->mtime = get_time(buf->st_mtime); base->inode_number = get_inode_no(dir_ent->inode); if(type == SQUASHFS_FILE_TYPE) { int i; struct squashfs_reg_inode_header *reg = &inode_header.reg; size_t off = offsetof(struct squashfs_reg_inode_header, block_list); inode = get_inode(sizeof(*reg) + offset * sizeof(unsigned int)); reg->file_size = byte_size; reg->start_block = start_block; reg->fragment = fragment->index; reg->offset = fragment->offset; SQUASHFS_SWAP_REG_INODE_HEADER(reg, inode); SQUASHFS_SWAP_INTS(block_list, inode + off, offset); TRACE("File inode, file_size %lld, start_block 0x%llx, blocks " "%d, fragment %d, offset %d, size %d\n", byte_size, start_block, offset, fragment->index, fragment->offset, fragment->size); for(i = 0; i < offset; i++) TRACE("Block %d, size %d\n", i, block_list[i]); } else if(type == SQUASHFS_LREG_TYPE) { int i; struct squashfs_lreg_inode_header *reg = &inode_header.lreg; size_t off = offsetof(struct squashfs_lreg_inode_header, block_list); inode = get_inode(sizeof(*reg) + offset * sizeof(unsigned int)); reg->nlink = nlink; reg->file_size = byte_size; reg->start_block = start_block; reg->fragment = fragment->index; reg->offset = fragment->offset; if(sparse && sparse >= byte_size) sparse = byte_size - 1; reg->sparse = sparse; reg->xattr = xattr; SQUASHFS_SWAP_LREG_INODE_HEADER(reg, inode); SQUASHFS_SWAP_INTS(block_list, inode + off, offset); TRACE("Long file inode, file_size %lld, start_block 0x%llx, " "blocks %d, fragment %d, offset %d, size %d, nlink %d" "\n", byte_size, start_block, offset, fragment->index, fragment->offset, fragment->size, nlink); for(i = 0; i < offset; i++) TRACE("Block %d, size %d\n", i, block_list[i]); } else if(type == SQUASHFS_LDIR_TYPE) { int i; unsigned char *p; struct squashfs_ldir_inode_header *dir = &inode_header.ldir; struct cached_dir_index *index = dir_in->index; unsigned int i_count = dir_in->i_count; unsigned int i_size = dir_in->i_size; if(byte_size >= 1LL << 32) BAD_ERROR("directory greater than 2^32-1 bytes!\n"); inode = get_inode(sizeof(*dir) + i_size); dir->inode_type = SQUASHFS_LDIR_TYPE; dir->nlink = dir_ent->dir->directory_count + 2; dir->file_size = byte_size; dir->offset = offset; dir->start_block = start_block; dir->i_count = i_count; dir->parent_inode = get_parent_no(dir_ent->our_dir); dir->xattr = xattr; SQUASHFS_SWAP_LDIR_INODE_HEADER(dir, inode); p = inode + offsetof(struct squashfs_ldir_inode_header, index); for(i = 0; i < i_count; i++) { SQUASHFS_SWAP_DIR_INDEX(&index[i].index, p); p += offsetof(struct squashfs_dir_index, name); memcpy(p, index[i].name, index[i].index.size + 1); p += index[i].index.size + 1; } TRACE("Long directory inode, file_size %lld, start_block " "0x%llx, offset 0x%x, nlink %d\n", byte_size, start_block, offset, dir_ent->dir->directory_count + 2); } else if(type == SQUASHFS_DIR_TYPE) { struct squashfs_dir_inode_header *dir = &inode_header.dir; inode = get_inode(sizeof(*dir)); dir->nlink = dir_ent->dir->directory_count + 2; dir->file_size = byte_size; dir->offset = offset; dir->start_block = start_block; dir->parent_inode = get_parent_no(dir_ent->our_dir); SQUASHFS_SWAP_DIR_INODE_HEADER(dir, inode); TRACE("Directory inode, file_size %lld, start_block 0x%llx, " "offset 0x%x, nlink %d\n", byte_size, start_block, offset, dir_ent->dir->directory_count + 2); } else if(type == SQUASHFS_CHRDEV_TYPE || type == SQUASHFS_BLKDEV_TYPE) { struct squashfs_dev_inode_header *dev = &inode_header.dev; unsigned int major = major(buf->st_rdev); unsigned int minor = minor(buf->st_rdev); if(major > 0xfff) { ERROR("Major %d out of range in device node %s, " "truncating to %d\n", major, filename, major & 0xfff); major &= 0xfff; } if(minor > 0xfffff) { ERROR("Minor %d out of range in device node %s, " "truncating to %d\n", minor, filename, minor & 0xfffff); minor &= 0xfffff; } inode = get_inode(sizeof(*dev)); dev->nlink = nlink; dev->rdev = (major << 8) | (minor & 0xff) | ((minor & ~0xff) << 12); SQUASHFS_SWAP_DEV_INODE_HEADER(dev, inode); TRACE("Device inode, rdev 0x%x, nlink %d\n", dev->rdev, nlink); } else if(type == SQUASHFS_LCHRDEV_TYPE || type == SQUASHFS_LBLKDEV_TYPE) { struct squashfs_ldev_inode_header *dev = &inode_header.ldev; unsigned int major = major(buf->st_rdev); unsigned int minor = minor(buf->st_rdev); if(major > 0xfff) { ERROR("Major %d out of range in device node %s, " "truncating to %d\n", major, filename, major & 0xfff); major &= 0xfff; } if(minor > 0xfffff) { ERROR("Minor %d out of range in device node %s, " "truncating to %d\n", minor, filename, minor & 0xfffff); minor &= 0xfffff; } inode = get_inode(sizeof(*dev)); dev->nlink = nlink; dev->rdev = (major << 8) | (minor & 0xff) | ((minor & ~0xff) << 12); dev->xattr = xattr; SQUASHFS_SWAP_LDEV_INODE_HEADER(dev, inode); TRACE("Device inode, rdev 0x%x, nlink %d\n", dev->rdev, nlink); } else if(type == SQUASHFS_SYMLINK_TYPE) { struct squashfs_symlink_inode_header *symlink = &inode_header.symlink; int byte = strlen(dir_ent->inode->symlink); size_t off = offsetof(struct squashfs_symlink_inode_header, symlink); inode = get_inode(sizeof(*symlink) + byte); symlink->nlink = nlink; symlink->symlink_size = byte; SQUASHFS_SWAP_SYMLINK_INODE_HEADER(symlink, inode); strncpy(inode + off, dir_ent->inode->symlink, byte); TRACE("Symbolic link inode, symlink_size %d, nlink %d\n", byte, nlink); } else if(type == SQUASHFS_LSYMLINK_TYPE) { struct squashfs_symlink_inode_header *symlink = &inode_header.symlink; int byte = strlen(dir_ent->inode->symlink); size_t off = offsetof(struct squashfs_symlink_inode_header, symlink); inode = get_inode(sizeof(*symlink) + byte + sizeof(unsigned int)); symlink->nlink = nlink; symlink->symlink_size = byte; SQUASHFS_SWAP_SYMLINK_INODE_HEADER(symlink, inode); strncpy(inode + off, dir_ent->inode->symlink, byte); SQUASHFS_SWAP_INTS(&xattr, inode + off + byte, 1); TRACE("Symbolic link inode, symlink_size %d, nlink %d\n", byte, nlink); } else if(type == SQUASHFS_FIFO_TYPE || type == SQUASHFS_SOCKET_TYPE) { struct squashfs_ipc_inode_header *ipc = &inode_header.ipc; inode = get_inode(sizeof(*ipc)); ipc->nlink = nlink; SQUASHFS_SWAP_IPC_INODE_HEADER(ipc, inode); TRACE("ipc inode, type %s, nlink %d\n", type == SQUASHFS_FIFO_TYPE ? "fifo" : "socket", nlink); } else if(type == SQUASHFS_LFIFO_TYPE || type == SQUASHFS_LSOCKET_TYPE) { struct squashfs_lipc_inode_header *ipc = &inode_header.lipc; inode = get_inode(sizeof(*ipc)); ipc->nlink = nlink; ipc->xattr = xattr; SQUASHFS_SWAP_LIPC_INODE_HEADER(ipc, inode); TRACE("ipc inode, type %s, nlink %d\n", type == SQUASHFS_FIFO_TYPE ? "fifo" : "socket", nlink); } else BAD_ERROR("Unrecognised inode %d in create_inode\n", type); inode_count ++; TRACE("Created inode 0x%llx, type %d, uid %d, guid %d\n", MKINODE(inode), type, base->uid, base->guid); return MKINODE(inode); } static void add_dir(squashfs_inode inode, unsigned int inode_number, char *name, int type, struct directory *dir) { unsigned char *buff; struct squashfs_dir_entry idir; unsigned int start_block = inode >> 16; unsigned int offset = inode & 0xffff; unsigned int size = strlen(name); size_t name_off = offsetof(struct squashfs_dir_entry, name); if(size > SQUASHFS_NAME_LEN) { size = SQUASHFS_NAME_LEN; ERROR("Filename is greater than %d characters, truncating! ..." "\n", SQUASHFS_NAME_LEN); } if(dir->p + sizeof(struct squashfs_dir_entry) + size + sizeof(struct squashfs_dir_header) >= dir->buff + dir->size) { buff = realloc(dir->buff, dir->size += SQUASHFS_METADATA_SIZE); if(buff == NULL) MEM_ERROR(); dir->p = (dir->p - dir->buff) + buff; if(dir->entry_count_p) dir->entry_count_p = (dir->entry_count_p - dir->buff + buff); dir->index_count_p = dir->index_count_p - dir->buff + buff; dir->buff = buff; } if(dir->entry_count == 256 || start_block != dir->start_block || ((dir->entry_count_p != NULL) && ((dir->p + sizeof(struct squashfs_dir_entry) + size - dir->index_count_p) > SQUASHFS_METADATA_SIZE)) || ((long long) inode_number - dir->inode_number) > 32767 || ((long long) inode_number - dir->inode_number) < -32768) { if(dir->entry_count_p) { struct squashfs_dir_header dir_header; if((dir->p + sizeof(struct squashfs_dir_entry) + size - dir->index_count_p) > SQUASHFS_METADATA_SIZE) { if(dir->i_count % I_COUNT_SIZE == 0) { dir->index = realloc(dir->index, (dir->i_count + I_COUNT_SIZE) * sizeof(struct cached_dir_index)); if(dir->index == NULL) MEM_ERROR(); } dir->index[dir->i_count].index.index = dir->p - dir->buff; dir->index[dir->i_count].index.size = size - 1; dir->index[dir->i_count++].name = name; dir->i_size += sizeof(struct squashfs_dir_index) + size; dir->index_count_p = dir->p; } dir_header.count = dir->entry_count - 1; dir_header.start_block = dir->start_block; dir_header.inode_number = dir->inode_number; SQUASHFS_SWAP_DIR_HEADER(&dir_header, dir->entry_count_p); } dir->entry_count_p = dir->p; dir->start_block = start_block; dir->entry_count = 0; dir->inode_number = inode_number; dir->p += sizeof(struct squashfs_dir_header); } idir.offset = offset; idir.type = type; idir.size = size - 1; idir.inode_number = ((long long) inode_number - dir->inode_number); SQUASHFS_SWAP_DIR_ENTRY(&idir, dir->p); strncpy((char *) dir->p + name_off, name, size); dir->p += sizeof(struct squashfs_dir_entry) + size; dir->entry_count ++; } static squashfs_inode write_dir(struct dir_info *dir_info, struct directory *dir) { long long dir_size = dir->p - dir->buff; int data_space = directory_cache_size - directory_cache_bytes; unsigned int directory_block, directory_offset, i_count, index; unsigned short c_byte; if(data_space < dir_size) { int realloc_size = directory_cache_size == 0 ? ((dir_size + SQUASHFS_METADATA_SIZE) & ~(SQUASHFS_METADATA_SIZE - 1)) : dir_size - data_space; void *dc = realloc(directory_data_cache, directory_cache_size + realloc_size); if(dc == NULL) MEM_ERROR(); directory_cache_size += realloc_size; directory_data_cache = dc; } if(dir_size) { struct squashfs_dir_header dir_header; dir_header.count = dir->entry_count - 1; dir_header.start_block = dir->start_block; dir_header.inode_number = dir->inode_number; SQUASHFS_SWAP_DIR_HEADER(&dir_header, dir->entry_count_p); memcpy(directory_data_cache + directory_cache_bytes, dir->buff, dir_size); } directory_offset = directory_cache_bytes; directory_block = directory_bytes; directory_cache_bytes += dir_size; i_count = 0; index = SQUASHFS_METADATA_SIZE - directory_offset; while(1) { while(i_count < dir->i_count && dir->index[i_count].index.index < index) dir->index[i_count++].index.start_block = directory_bytes; index += SQUASHFS_METADATA_SIZE; if(directory_cache_bytes < SQUASHFS_METADATA_SIZE) break; if((directory_size - directory_bytes) < ((SQUASHFS_METADATA_SIZE << 1) + 2)) { void *dt = realloc(directory_table, directory_size + (SQUASHFS_METADATA_SIZE << 1) + 2); if(dt == NULL) MEM_ERROR(); directory_size += SQUASHFS_METADATA_SIZE << 1; directory_table = dt; } c_byte = mangle(directory_table + directory_bytes + BLOCK_OFFSET, directory_data_cache, SQUASHFS_METADATA_SIZE, SQUASHFS_METADATA_SIZE, noI, 0); TRACE("Directory block @ 0x%x, size %d\n", directory_bytes, c_byte); SQUASHFS_SWAP_SHORTS(&c_byte, directory_table + directory_bytes, 1); directory_bytes += SQUASHFS_COMPRESSED_SIZE(c_byte) + BLOCK_OFFSET; total_directory_bytes += SQUASHFS_METADATA_SIZE + BLOCK_OFFSET; memmove(directory_data_cache, directory_data_cache + SQUASHFS_METADATA_SIZE, directory_cache_bytes - SQUASHFS_METADATA_SIZE); directory_cache_bytes -= SQUASHFS_METADATA_SIZE; } dir_count ++; #ifndef SQUASHFS_TRACE return create_inode(dir_info, dir_info->dir_ent, SQUASHFS_DIR_TYPE, dir_size + 3, directory_block, directory_offset, NULL, NULL, dir, 0); #else { unsigned char *dirp; int count; squashfs_inode inode; inode = create_inode(dir_info, dir_info->dir_ent, SQUASHFS_DIR_TYPE, dir_size + 3, directory_block, directory_offset, NULL, NULL, dir, 0); TRACE("Directory contents of inode 0x%llx\n", inode); dirp = dir->buff; while(dirp < dir->p) { char buffer[SQUASHFS_NAME_LEN + 1]; struct squashfs_dir_entry idir, *idirp; struct squashfs_dir_header dirh; SQUASHFS_SWAP_DIR_HEADER((struct squashfs_dir_header *) dirp, &dirh); count = dirh.count + 1; dirp += sizeof(struct squashfs_dir_header); TRACE("\tStart block 0x%x, count %d\n", dirh.start_block, count); while(count--) { idirp = (struct squashfs_dir_entry *) dirp; SQUASHFS_SWAP_DIR_ENTRY(idirp, &idir); strncpy(buffer, idirp->name, idir.size + 1); buffer[idir.size + 1] = '\0'; TRACE("\t\tname %s, inode offset 0x%x, type " "%d\n", buffer, idir.offset, idir.type); dirp += sizeof(struct squashfs_dir_entry) + idir.size + 1; } } return inode; } #endif } static struct file_buffer *get_fragment(struct fragment *fragment) { struct squashfs_fragment_entry *disk_fragment; struct file_buffer *buffer, *compressed_buffer; long long start_block; int res, size, index = fragment->index, compressed; char locked; /* * Lookup fragment block in cache. * If the fragment block doesn't exist, then get the compressed version * from the writer cache or off disk, and decompress it. * * This routine has two things which complicate the code: * * 1. Multiple threads can simultaneously lookup/create the * same buffer. This means a buffer needs to be "locked" * when it is being filled in, to prevent other threads from * using it when it is not ready. This is because we now do * fragment duplicate checking in parallel. * 2. We have two caches which need to be checked for the * presence of fragment blocks: the normal fragment cache * and a "reserve" cache. The reserve cache is used to * prevent an unnecessary pipeline stall when the fragment cache * is full of fragments waiting to be compressed. */ if(fragment->index == SQUASHFS_INVALID_FRAG) return NULL; pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); pthread_mutex_lock(&dup_mutex); again: buffer = cache_lookup_nowait(fragment_buffer, index, &locked); if(buffer) { pthread_mutex_unlock(&dup_mutex); if(locked) /* got a buffer being filled in. Wait for it */ cache_wait_unlock(buffer); goto finished; } /* not in fragment cache, is it in the reserve cache? */ buffer = cache_lookup_nowait(reserve_cache, index, &locked); if(buffer) { pthread_mutex_unlock(&dup_mutex); if(locked) /* got a buffer being filled in. Wait for it */ cache_wait_unlock(buffer); goto finished; } /* in neither cache, try to get it from the fragment cache */ buffer = cache_get_nowait(fragment_buffer, index); if(!buffer) { /* * no room, get it from the reserve cache, this is * dimensioned so it will always have space (no more than * processors + 1 can have an outstanding reserve buffer) */ buffer = cache_get_nowait(reserve_cache, index); if(!buffer) { /* failsafe */ ERROR("no space in reserve cache\n"); goto again; } } pthread_mutex_unlock(&dup_mutex); compressed_buffer = cache_lookup(fwriter_buffer, index); pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); pthread_mutex_lock(&fragment_mutex); disk_fragment = &fragment_table[index]; size = SQUASHFS_COMPRESSED_SIZE_BLOCK(disk_fragment->size); compressed = SQUASHFS_COMPRESSED_BLOCK(disk_fragment->size); start_block = disk_fragment->start_block; pthread_cleanup_pop(1); if(compressed) { int error; char *data; if(compressed_buffer) data = compressed_buffer->data; else { data = read_from_disk(start_block, size); if(data == NULL) { ERROR("Failed to read fragment from output" " filesystem\n"); BAD_ERROR("Output filesystem corrupted?\n"); } } res = compressor_uncompress(comp, buffer->data, data, size, block_size, &error); if(res == -1) BAD_ERROR("%s uncompress failed with error code %d\n", comp->name, error); } else if(compressed_buffer) memcpy(buffer->data, compressed_buffer->data, size); else { res = read_fs_bytes(fd, start_block, size, buffer->data); if(res == 0) { ERROR("Failed to read fragment from output " "filesystem\n"); BAD_ERROR("Output filesystem corrupted?\n"); } } cache_unlock(buffer); cache_block_put(compressed_buffer); finished: pthread_cleanup_pop(0); return buffer; } static unsigned short get_fragment_checksum(struct file_info *file) { struct file_buffer *frag_buffer; struct append_file *append; int res, index = file->fragment->index; unsigned short checksum; if(index == SQUASHFS_INVALID_FRAG) return 0; pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); pthread_mutex_lock(&dup_mutex); res = file->have_frag_checksum; checksum = file->fragment_checksum; pthread_cleanup_pop(1); if(res) return checksum; frag_buffer = get_fragment(file->fragment); pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); for(append = file_mapping[index]; append; append = append->next) { int offset = append->file->fragment->offset; int size = append->file->fragment->size; unsigned short cksum = get_checksum_mem(frag_buffer->data + offset, size); if(file == append->file) checksum = cksum; pthread_mutex_lock(&dup_mutex); append->file->fragment_checksum = cksum; append->file->have_frag_checksum = TRUE; pthread_mutex_unlock(&dup_mutex); } cache_block_put(frag_buffer); pthread_cleanup_pop(0); return checksum; } static void ensure_fragments_flushed() { pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); pthread_mutex_lock(&fragment_mutex); while(fragments_outstanding) pthread_cond_wait(&fragment_waiting, &fragment_mutex); pthread_cleanup_pop(1); } static void lock_fragments() { pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); pthread_mutex_lock(&fragment_mutex); fragments_locked = TRUE; pthread_cleanup_pop(1); } static void log_fragment(unsigned int fragment, long long start) { if(logging) fprintf(log_fd, "Fragment %u, %lld\n", fragment, start); } static void unlock_fragments() { int frg, size; struct file_buffer *write_buffer; pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); pthread_mutex_lock(&fragment_mutex); /* * Note queue_empty() is inherently racy with respect to concurrent * queue get and pushes. We avoid this because we're holding the * fragment_mutex which ensures no other threads can be using the * queue at this time. */ while(!queue_empty(locked_fragment)) { write_buffer = queue_get(locked_fragment); frg = write_buffer->block; size = SQUASHFS_COMPRESSED_SIZE_BLOCK(fragment_table[frg].size); fragment_table[frg].start_block = bytes; write_buffer->block = bytes; bytes += size; fragments_outstanding --; queue_put(to_writer, write_buffer); log_fragment(frg, fragment_table[frg].start_block); TRACE("fragment_locked writing fragment %d, compressed size %d" "\n", frg, size); } fragments_locked = FALSE; pthread_cleanup_pop(1); } /* Called with the fragment_mutex locked */ static void add_pending_fragment(struct file_buffer *write_buffer, int c_byte, int fragment) { fragment_table[fragment].size = c_byte; write_buffer->block = fragment; queue_put(locked_fragment, write_buffer); } static void write_fragment(struct file_buffer *fragment) { static long long sequence = 0; if(fragment == NULL) return; pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); pthread_mutex_lock(&fragment_mutex); fragment_table[fragment->block].unused = 0; fragment->sequence = sequence ++; fragments_outstanding ++; queue_put(to_frag, fragment); pthread_cleanup_pop(1); } static struct file_buffer *allocate_fragment() { struct file_buffer *fragment = cache_get(fragment_buffer, fragments); pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); pthread_mutex_lock(&fragment_mutex); if(fragments % FRAG_SIZE == 0) { void *ft = realloc(fragment_table, (fragments + FRAG_SIZE) * sizeof(struct squashfs_fragment_entry)); if(ft == NULL) MEM_ERROR(); fragment_table = ft; } fragment->size = 0; fragment->block = fragments ++; pthread_cleanup_pop(1); return fragment; } static struct fragment empty_fragment = {SQUASHFS_INVALID_FRAG, 0, 0}; void free_fragment(struct fragment *fragment) { if(fragment != &empty_fragment) free(fragment); } static struct fragment *get_and_fill_fragment(struct file_buffer *file_buffer, struct dir_ent *dir_ent, int tail) { struct fragment *ffrg; struct file_buffer **fragment; if(file_buffer == NULL || file_buffer->size == 0) return &empty_fragment; fragment = eval_frag_actions(root_dir, dir_ent, tail); if((*fragment) && (*fragment)->size + file_buffer->size > block_size) { write_fragment(*fragment); *fragment = NULL; } ffrg = malloc(sizeof(struct fragment)); if(ffrg == NULL) MEM_ERROR(); if(*fragment == NULL) *fragment = allocate_fragment(); ffrg->index = (*fragment)->block; ffrg->offset = (*fragment)->size; ffrg->size = file_buffer->size; memcpy((*fragment)->data + (*fragment)->size, file_buffer->data, file_buffer->size); (*fragment)->size += file_buffer->size; return ffrg; } long long generic_write_table(long long length, void *buffer, int length2, void *buffer2, int uncompressed) { int meta_blocks = (length + SQUASHFS_METADATA_SIZE - 1) / SQUASHFS_METADATA_SIZE; long long *list, start_bytes; int compressed_size, i, list_size = meta_blocks * sizeof(long long); unsigned short c_byte; char cbuffer[(SQUASHFS_METADATA_SIZE << 2) + 2]; #ifdef SQUASHFS_TRACE long long obytes = bytes; long long olength = length; #endif list = malloc(list_size); if(list == NULL) MEM_ERROR(); for(i = 0; i < meta_blocks; i++) { int avail_bytes = length > SQUASHFS_METADATA_SIZE ? SQUASHFS_METADATA_SIZE : length; c_byte = mangle(cbuffer + BLOCK_OFFSET, buffer + i * SQUASHFS_METADATA_SIZE , avail_bytes, SQUASHFS_METADATA_SIZE, uncompressed, 0); SQUASHFS_SWAP_SHORTS(&c_byte, cbuffer, 1); list[i] = bytes; compressed_size = SQUASHFS_COMPRESSED_SIZE(c_byte) + BLOCK_OFFSET; TRACE("block %d @ 0x%llx, compressed size %d\n", i, bytes, compressed_size); write_destination(fd, bytes, compressed_size, cbuffer); bytes += compressed_size; total_bytes += avail_bytes; length -= avail_bytes; } start_bytes = bytes; if(length2) { write_destination(fd, bytes, length2, buffer2); bytes += length2; total_bytes += length2; } SQUASHFS_INSWAP_LONG_LONGS(list, meta_blocks); write_destination(fd, bytes, list_size, list); bytes += list_size; total_bytes += list_size; TRACE("generic_write_table: total uncompressed %lld compressed %lld\n", olength, bytes - obytes); free(list); return start_bytes; } static long long write_fragment_table() { long long frag_bytes = SQUASHFS_FRAGMENT_BYTES(fragments); unsigned int i; TRACE("write_fragment_table: fragments %u, frag_bytes %d\n", fragments, frag_bytes); for(i = 0; i < fragments; i++) { TRACE("write_fragment_table: fragment %u, start_block 0x%llx, " "size %d\n", i, fragment_table[i].start_block, fragment_table[i].size); SQUASHFS_INSWAP_FRAGMENT_ENTRY(&fragment_table[i]); } return generic_write_table(frag_bytes, fragment_table, 0, NULL, noF); } char read_from_file_buffer[SQUASHFS_FILE_MAX_SIZE]; static char *read_from_disk(long long start, unsigned int avail_bytes) { int res; res = read_fs_bytes(fd, start, avail_bytes, read_from_file_buffer); if(res == 0) return NULL; return read_from_file_buffer; } char read_from_file_buffer2[SQUASHFS_FILE_MAX_SIZE]; static char *read_from_disk2(long long start, unsigned int avail_bytes) { int res; res = read_fs_bytes(fd, start, avail_bytes, read_from_file_buffer2); if(res == 0) return NULL; return read_from_file_buffer2; } /* * Compute 16 bit BSD checksum over the data */ unsigned short get_checksum(char *buff, int bytes, unsigned short chksum) { unsigned char *b = (unsigned char *) buff; while(bytes --) { chksum = (chksum & 1) ? (chksum >> 1) | 0x8000 : chksum >> 1; chksum += *b++; } return chksum; } static unsigned short get_checksum_disk(long long start, long long l, unsigned int *blocks) { unsigned short chksum = 0; unsigned int bytes; struct file_buffer *write_buffer; int i; for(i = 0; l; i++) { bytes = SQUASHFS_COMPRESSED_SIZE_BLOCK(blocks[i]); if(bytes == 0) /* sparse block */ continue; write_buffer = cache_lookup(bwriter_buffer, start); if(write_buffer) { chksum = get_checksum(write_buffer->data, bytes, chksum); cache_block_put(write_buffer); } else { void *data = read_from_disk(start, bytes); if(data == NULL) { ERROR("Failed to checksum data from output" " filesystem\n"); BAD_ERROR("Output filesystem corrupted?\n"); } chksum = get_checksum(data, bytes, chksum); } l -= bytes; start += bytes; } return chksum; } unsigned short get_checksum_mem(char *buff, int bytes) { return get_checksum(buff, bytes, 0); } static int block_hash(int size, int blocks) { return ((size << 10) & 0xffc00) | (blocks & 0x3ff); } void add_file(long long start, long long file_size, long long file_bytes, unsigned int *block_listp, int blocks, unsigned int fragment, int offset, int bytes) { struct fragment *frg; unsigned int *block_list = block_listp; struct file_info *dupl_ptr; struct append_file *append_file; struct file_info *file; int blocks_dup = FALSE, frag_dup = FALSE; int bl_hash = 0; if(!duplicate_checking || file_size == 0) return; if(blocks) { bl_hash = block_hash(block_list[0], blocks); dupl_ptr = dupl_block[bl_hash]; for(; dupl_ptr; dupl_ptr = dupl_ptr->block_next) { if(start == dupl_ptr->start) break; } if(dupl_ptr) { /* * Our blocks have already been added. If we don't * have a fragment, then we've finished checking */ if(fragment == SQUASHFS_INVALID_FRAG) return; /* * This entry probably created both the blocks and * the tail-end fragment, and so check for that */ if((fragment == dupl_ptr->fragment->index) && (offset == dupl_ptr->fragment->offset) && (bytes == dupl_ptr->fragment->size)) return; /* * Remember our blocks are duplicate, and continue * looking for the tail-end fragment */ blocks_dup = TRUE; } } if(fragment != SQUASHFS_INVALID_FRAG) { dupl_ptr = dupl_frag[bytes]; for(; dupl_ptr; dupl_ptr = dupl_ptr->frag_next) if((fragment == dupl_ptr->fragment->index) && (offset == dupl_ptr->fragment->offset) && (bytes == dupl_ptr->fragment->size)) break; if(dupl_ptr) { /* * Our tail-end fragment entry has already been added. * If there's no blocks or they're dup, then we're done * here */ if(blocks == 0 || blocks_dup) return; /* Remember our tail-end fragment entry is duplicate */ frag_dup = TRUE; } } frg = malloc(sizeof(struct fragment)); if(frg == NULL) MEM_ERROR(); frg->index = fragment; frg->offset = offset; frg->size = bytes; file = add_non_dup(file_size, file_bytes, blocks, 0, block_list, start, frg, 0, 0, FALSE, FALSE, blocks_dup, frag_dup, bl_hash); if(fragment == SQUASHFS_INVALID_FRAG) return; append_file = malloc(sizeof(struct append_file)); if(append_file == NULL) MEM_ERROR(); append_file->file = file; append_file->next = file_mapping[fragment]; file_mapping[fragment] = append_file; } static int pre_duplicate(long long file_size, struct inode_info *inode, struct file_buffer *buffer, int *bl_hash) { struct file_info *dupl_ptr; long long fragment_size; int blocks; if(inode->no_fragments || (!inode->always_use_fragments && file_size >= block_size)) { blocks = (file_size + block_size - 1) >> block_log; fragment_size = 0; } else { blocks = file_size >> block_log; fragment_size = file_size & (block_size - 1); } /* Look for a possible duplicate set of blocks */ if(blocks) { *bl_hash = block_hash(buffer->size, blocks); for(dupl_ptr = dupl_block[*bl_hash]; dupl_ptr;dupl_ptr = dupl_ptr->block_next) if(dupl_ptr->blocks == blocks && dupl_ptr->block_list[0] == buffer->c_byte) return TRUE; } /* Look for a possible duplicate fragment */ if(fragment_size) { for(dupl_ptr = dupl_frag[fragment_size]; dupl_ptr; dupl_ptr = dupl_ptr->frag_next) if(dupl_ptr->fragment->size == fragment_size) return TRUE; } return FALSE; } static struct file_info *create_non_dup(long long file_size, long long bytes, unsigned int blocks, long long sparse, unsigned int *block_list, long long start,struct fragment *fragment,unsigned short checksum, unsigned short fragment_checksum, int checksum_flag, int checksum_frag_flag) { struct file_info *dupl_ptr = malloc(sizeof(struct file_info)); if(dupl_ptr == NULL) MEM_ERROR(); dupl_ptr->file_size = file_size; dupl_ptr->bytes = bytes; dupl_ptr->blocks = blocks; dupl_ptr->sparse = sparse; dupl_ptr->block_list = block_list; dupl_ptr->start = start; dupl_ptr->fragment = fragment; dupl_ptr->checksum = checksum; dupl_ptr->fragment_checksum = fragment_checksum; dupl_ptr->have_frag_checksum = checksum_frag_flag; dupl_ptr->have_checksum = checksum_flag; dupl_ptr->block_next = NULL; dupl_ptr->frag_next = NULL; dupl_ptr->dup = NULL; return dupl_ptr; } static struct file_info *add_non_dup(long long file_size, long long bytes, unsigned int blocks, long long sparse, unsigned int *block_list, long long start,struct fragment *fragment,unsigned short checksum, unsigned short fragment_checksum, int checksum_flag, int checksum_frag_flag, int blocks_dup, int frag_dup, int bl_hash) { struct file_info *dupl_ptr = malloc(sizeof(struct file_info)); int fragment_size = fragment->size; if(dupl_ptr == NULL) MEM_ERROR(); dupl_ptr->file_size = file_size; dupl_ptr->bytes = bytes; dupl_ptr->blocks = blocks; dupl_ptr->sparse = sparse; dupl_ptr->block_list = block_list; dupl_ptr->start = start; dupl_ptr->fragment = fragment; dupl_ptr->checksum = checksum; dupl_ptr->fragment_checksum = fragment_checksum; dupl_ptr->have_frag_checksum = checksum_frag_flag; dupl_ptr->have_checksum = checksum_flag; dupl_ptr->block_next = NULL; dupl_ptr->frag_next = NULL; dupl_ptr->dup = NULL; pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); pthread_mutex_lock(&dup_mutex); if(blocks && !blocks_dup) { dupl_ptr->block_next = dupl_block[bl_hash]; dupl_block[bl_hash] = dupl_ptr; } if(fragment_size && !frag_dup) { dupl_ptr->frag_next = dupl_frag[fragment_size]; dupl_frag[fragment_size] = dupl_ptr; } dup_files ++; pthread_cleanup_pop(1); return dupl_ptr; } static struct file_info *frag_duplicate(struct file_buffer *file_buffer, int *duplicate) { struct file_info *dupl_ptr; struct file_buffer *buffer; struct file_info *dupl_start = file_buffer->dupl_start; long long file_size = file_buffer->file_size; unsigned short checksum = file_buffer->checksum; int res; if(file_buffer->duplicate) dupl_ptr = dupl_start; else { for(dupl_ptr = dupl_frag[file_size]; dupl_ptr && dupl_ptr != dupl_start; dupl_ptr = dupl_ptr->frag_next) { if(file_size == dupl_ptr->fragment->size) { if(get_fragment_checksum(dupl_ptr) == checksum) { buffer = get_fragment(dupl_ptr->fragment); res = memcmp(file_buffer->data, buffer->data + dupl_ptr->fragment->offset, file_size); cache_block_put(buffer); if(res == 0) break; } } } if(!dupl_ptr || dupl_ptr == dupl_start) { *duplicate = FALSE; return NULL; } } if(dupl_ptr->file_size == file_size) { /* File only has a fragment, and so this is an exact match */ TRACE("Found duplicate file, fragment %u, size %d, offset %d, " "checksum 0x%x\n", dupl_ptr->fragment->index, file_size, dupl_ptr->fragment->offset, checksum); *duplicate = TRUE; return dupl_ptr; } else { struct dup_info *dup; /* * File also has a block list. Create a new file without * a block_list, and link it to this file. First check whether * it is already there. */ if(dupl_ptr->dup) { *duplicate = TRUE; return dupl_ptr->dup->file; } dup = malloc(sizeof(struct dup_info)); if(dup == NULL) MEM_ERROR(); dup->file = create_non_dup(file_size, 0, 0, 0, NULL, 0, dupl_ptr->fragment, 0, checksum, TRUE, TRUE); dup->next = NULL; dupl_ptr->dup = dup; *duplicate = FALSE; return dup->file; } } static struct file_info *duplicate(int *dupf, int *block_dup, long long file_size, long long bytes, unsigned int *block_list, long long start, struct dir_ent *dir_ent, struct file_buffer *file_buffer, int blocks, long long sparse, int bl_hash) { struct file_info *dupl_ptr, *file; struct file_info *block_dupl = NULL, *frag_dupl = NULL; struct dup_info *dup; int frag_bytes = file_buffer ? file_buffer->size : 0; unsigned short fragment_checksum = file_buffer ? file_buffer->checksum : 0; unsigned short checksum = 0; char checksum_flag = FALSE; struct fragment *fragment; /* Look for a possible duplicate set of blocks */ for(dupl_ptr = dupl_block[bl_hash]; dupl_ptr; dupl_ptr = dupl_ptr->block_next) { if(bytes == dupl_ptr->bytes && blocks == dupl_ptr->blocks) { long long target_start, dup_start = dupl_ptr->start; int block; /* * Block list has same uncompressed size and same * compressed size. Now check if each block compressed * to the same size */ if(memcmp(block_list, dupl_ptr->block_list, blocks * sizeof(unsigned int)) != 0) continue; /* Now get the checksums and compare */ if(checksum_flag == FALSE) { checksum = get_checksum_disk(start, bytes, block_list); checksum_flag = TRUE; } if(!dupl_ptr->have_checksum) { dupl_ptr->checksum = get_checksum_disk(dupl_ptr->start, dupl_ptr->bytes, dupl_ptr->block_list); dupl_ptr->have_checksum = TRUE; } if(checksum != dupl_ptr->checksum) continue; /* * Checksums match, so now we need to do a byte by byte * comparison */ target_start = start; for(block = 0; block < blocks; block ++) { int size = SQUASHFS_COMPRESSED_SIZE_BLOCK(block_list[block]); struct file_buffer *target_buffer = NULL; struct file_buffer *dup_buffer = NULL; char *target_data, *dup_data; int res; /* Sparse blocks obviously match */ if(size == 0) continue; /* * Get the block for our file. This will be in * the cache unless the cache wasn't large * enough to hold the entire file, in which case * the block will have been written to disk. */ target_buffer = cache_lookup(bwriter_buffer, target_start); if(target_buffer) target_data = target_buffer->data; else { target_data = read_from_disk(target_start, size); if(target_data == NULL) { ERROR("Failed to read data from" " output filesystem\n"); BAD_ERROR("Output filesystem" " corrupted?\n"); } } /* * Get the block for the other file. This may * still be in the cache (if it was written * recently), otherwise it will have to be read * back from disk */ dup_buffer = cache_lookup(bwriter_buffer, dup_start); if(dup_buffer) dup_data = dup_buffer->data; else { dup_data = read_from_disk2(dup_start, size); if(dup_data == NULL) { ERROR("Failed to read data from" " output filesystem\n"); BAD_ERROR("Output filesystem" " corrupted?\n"); } } res = memcmp(target_data, dup_data, size); cache_block_put(target_buffer); cache_block_put(dup_buffer); if(res != 0) break; target_start += size; dup_start += size; } if(block != blocks) continue; /* * Yes, the block list matches. We can use this, rather * than writing an identical block list. * If both it and us doesn't have a tail-end fragment * then we're finished. Return the duplicate. * * We have to deal with the special case where the * last block is a sparse block. This means the * file will have matched, but, it may be a different * file length (because a tail-end sparse block may be * anything from 1 byte to block_size - 1 in size, but * stored as zero). We can still use the block list in * this case, but, we must return a new entry with the * correct file size */ if(!frag_bytes && !dupl_ptr->fragment->size) { *dupf = *block_dup = TRUE; if(file_size == dupl_ptr->file_size) return dupl_ptr; else return create_non_dup(file_size, bytes, blocks, sparse, dupl_ptr->block_list, dupl_ptr->start, dupl_ptr->fragment, checksum, 0, checksum_flag, FALSE); } /* * We've got a tail-end fragment, and this file most * likely has a matching tail-end fragment (i.e. it is * a completely duplicate file). So save time and have * a look now. */ if(frag_bytes == dupl_ptr->fragment->size && fragment_checksum == get_fragment_checksum(dupl_ptr)) { /* * Checksums match, so now we need to do a byte * by byte comparison * */ struct file_buffer *frag_buffer = get_fragment(dupl_ptr->fragment); int res = memcmp(file_buffer->data, frag_buffer->data + dupl_ptr->fragment->offset, frag_bytes); cache_block_put(frag_buffer); if(res == 0) { /* * Yes, the fragment matches. We're now * finished. Return the duplicate */ *dupf = *block_dup = TRUE; return dupl_ptr; } } /* * No, the fragment didn't match. Remember the file * with the matching blocks, and look for a matching * fragment in the fragment list */ block_dupl = dupl_ptr; break; } } /* Look for a possible duplicate fragment */ if(frag_bytes) { for(dupl_ptr = dupl_frag[frag_bytes]; dupl_ptr; dupl_ptr = dupl_ptr->frag_next) { if(frag_bytes == dupl_ptr->fragment->size && fragment_checksum == get_fragment_checksum(dupl_ptr)) { /* * Checksums match, so now we need to do a byte * by byte comparison */ struct file_buffer *frag_buffer = get_fragment(dupl_ptr->fragment); int res = memcmp(file_buffer->data, frag_buffer->data + dupl_ptr->fragment->offset, frag_bytes); cache_block_put(frag_buffer); if(res == 0) { /* * Yes, the fragment matches. This file * may have a matching block list and * fragment, in which case we're * finished. */ if(block_dupl && block_dupl->start == dupl_ptr->start) { *dupf = *block_dup = TRUE; return dupl_ptr; } /* * Block list doesn't match. We can * construct a hybrid from these two * partially matching files */ frag_dupl = dupl_ptr; break; } } } } /* * If we've got here, then we've either matched on nothing, or got a * partial match. Matched on nothing is straightforward */ if(!block_dupl && !frag_dupl) { *dupf = *block_dup = FALSE; fragment = get_and_fill_fragment(file_buffer, dir_ent, TRUE); return add_non_dup(file_size, bytes, blocks, sparse, block_list, start, fragment, checksum, fragment_checksum, checksum_flag, file_buffer != NULL, FALSE, FALSE, bl_hash); } /* * At this point, we may have * 1. A partially matching single file. For example the file may * contain the block list we want, but, it has the wrong tail-end, * or vice-versa, * 2. A partially matching single file for another reason. For example * it has the block list we want, and a tail-end, whereas we don't * have a tail-end. Note the vice-versa situation doesn't appear * here (it is handled in frag_duplicate). * 3. We have two partially matching files. One has the block list we * want, and the other has the tail-end we want. * * Strictly speaking, a file which is constructed from one or two * partial matches isn't a duplicate (of any single file), and it will * be confusing to list it as such (using the -info option). But a * second and thereafter appearance of this combination *is* a * duplicate of another file. Some of this second and thereafter * appearance is already handled above */ if(block_dupl && (!frag_bytes || frag_dupl)) { /* * This file won't be added to any hash list, because it is a * complete duplicate, and it doesn't need extra data to be * stored, e.g. part 2 & 3 above. So keep track of it by adding * it to a linked list. Obviously check if it's already there * first. */ for(dup = block_dupl->dup; dup; dup = dup->next) if((!frag_bytes && dup->frag == NULL) || (frag_bytes && dup->frag == frag_dupl)) break; if(dup) { /* Found a matching file. Return the duplicate */ *dupf = *block_dup = TRUE; return dup->file; } } if(frag_dupl) fragment = frag_dupl->fragment; else fragment = get_and_fill_fragment(file_buffer, dir_ent, TRUE); if(block_dupl) { start = block_dupl->start; block_list = block_dupl->block_list; } *dupf = FALSE; *block_dup = block_dupl != NULL; file = create_non_dup(file_size, bytes, blocks, sparse, block_list, start, fragment, checksum, fragment_checksum, checksum_flag, file_buffer != NULL); if(!block_dupl || (frag_bytes && !frag_dupl)) { /* * Partial duplicate, had to store some extra data for this * file, either a block list, or a fragment */ pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); pthread_mutex_lock(&dup_mutex); if(!block_dupl) { file->block_next = dupl_block[bl_hash]; dupl_block[bl_hash] = file; } if(frag_bytes && !frag_dupl) { file->frag_next = dupl_frag[frag_bytes]; dupl_frag[frag_bytes] = file; } dup_files ++; pthread_cleanup_pop(1); } else { dup = malloc(sizeof(struct dup_info)); if(dup == NULL) MEM_ERROR(); dup->frag = frag_dupl; dup->file = file; dup->next = block_dupl->dup; block_dupl->dup = dup; } return file; } static void *writer(void *arg) { while(1) { struct file_buffer *file_buffer = queue_get(to_writer); off_t off; if(file_buffer == NULL) { queue_put(from_writer, NULL); continue; } off = file_buffer->block; pthread_cleanup_push((void *) pthread_mutex_unlock, &pos_mutex); pthread_mutex_lock(&pos_mutex); if(lseek(fd, start_offset + off, SEEK_SET) == -1) { ERROR("writer: Lseek on destination failed because " "%s, offset=0x%llx\n", strerror(errno), start_offset + off); BAD_ERROR("Probably out of space on output " "%s\n", block_device ? "block device" : "filesystem"); } if(write_bytes(fd, file_buffer->data, file_buffer->size) == -1) BAD_ERROR("Failed to write to output %s\n", block_device ? "block device" : "filesystem"); pthread_cleanup_pop(1); cache_block_put(file_buffer); } } static int all_zero(struct file_buffer *file_buffer) { int i; long entries = file_buffer->size / sizeof(long); long *p = (long *) file_buffer->data; for(i = 0; i < entries && p[i] == 0; i++); if(i == entries) { for(i = file_buffer->size & ~(sizeof(long) - 1); i < file_buffer->size && file_buffer->data[i] == 0; i++); return i == file_buffer->size; } return 0; } static void *deflator(void *arg) { struct file_buffer *write_buffer = cache_get_nohash(bwriter_buffer); void *stream = NULL; int res; res = compressor_init(comp, &stream, block_size, 1); if(res) BAD_ERROR("deflator:: compressor_init failed\n"); while(1) { struct file_buffer *file_buffer = queue_get(to_deflate); if(sparse_files && all_zero(file_buffer)) { file_buffer->c_byte = 0; seq_queue_put(to_main, file_buffer); } else { write_buffer->c_byte = mangle2(stream, write_buffer->data, file_buffer->data, file_buffer->size, block_size, file_buffer->noD, 1); write_buffer->sequence = file_buffer->sequence; write_buffer->file_size = file_buffer->file_size; write_buffer->block = file_buffer->block; write_buffer->size = SQUASHFS_COMPRESSED_SIZE_BLOCK (write_buffer->c_byte); write_buffer->fragment = FALSE; write_buffer->error = FALSE; cache_block_put(file_buffer); seq_queue_put(to_main, write_buffer); write_buffer = cache_get_nohash(bwriter_buffer); } } } static void *frag_deflator(void *arg) { void *stream = NULL; int res; res = compressor_init(comp, &stream, block_size, 1); if(res) BAD_ERROR("frag_deflator:: compressor_init failed\n"); pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); while(1) { int c_byte, compressed_size; struct file_buffer *file_buffer = queue_get(to_frag); struct file_buffer *write_buffer = cache_get(fwriter_buffer, file_buffer->block); c_byte = mangle2(stream, write_buffer->data, file_buffer->data, file_buffer->size, block_size, noF, 1); compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte); write_buffer->size = compressed_size; pthread_mutex_lock(&fragment_mutex); if(fragments_locked == FALSE) { fragment_table[file_buffer->block].size = c_byte; fragment_table[file_buffer->block].start_block = bytes; write_buffer->block = bytes; bytes += compressed_size; fragments_outstanding --; queue_put(to_writer, write_buffer); log_fragment(file_buffer->block, fragment_table[file_buffer->block].start_block); pthread_mutex_unlock(&fragment_mutex); TRACE("Writing fragment %lld, uncompressed size %d, " "compressed size %d\n", file_buffer->block, file_buffer->size, compressed_size); } else { add_pending_fragment(write_buffer, c_byte, file_buffer->block); pthread_mutex_unlock(&fragment_mutex); } cache_block_put(file_buffer); } pthread_cleanup_pop(0); return NULL; } static void *frag_order_deflator(void *arg) { void *stream = NULL; int res; res = compressor_init(comp, &stream, block_size, 1); if(res) BAD_ERROR("frag_deflator:: compressor_init failed\n"); while(1) { int c_byte; struct file_buffer *file_buffer = queue_get(to_frag); struct file_buffer *write_buffer = cache_get(fwriter_buffer, file_buffer->block); c_byte = mangle2(stream, write_buffer->data, file_buffer->data, file_buffer->size, block_size, noF, 1); write_buffer->block = file_buffer->block; write_buffer->sequence = file_buffer->sequence; write_buffer->size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte); write_buffer->fragment = FALSE; pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); pthread_mutex_lock(&fragment_mutex); fragment_table[file_buffer->block].size = c_byte; pthread_cleanup_pop(1); seq_queue_put(to_order, write_buffer); TRACE("Writing fragment %lld, uncompressed size %d, " "compressed size %d\n", file_buffer->block, file_buffer->size, SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte)); cache_block_put(file_buffer); } } static void *frag_orderer(void *arg) { pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); while(1) { struct file_buffer *write_buffer = seq_queue_get(to_order); int block = write_buffer->block; pthread_mutex_lock(&fragment_mutex); fragment_table[block].start_block = bytes; write_buffer->block = bytes; bytes += SQUASHFS_COMPRESSED_SIZE_BLOCK(write_buffer->size); fragments_outstanding --; log_fragment(block, write_buffer->block); queue_put(to_writer, write_buffer); pthread_cond_signal(&fragment_waiting); pthread_mutex_unlock(&fragment_mutex); } pthread_cleanup_pop(0); return NULL; } static struct file_buffer *get_file_buffer() { struct file_buffer *file_buffer = seq_queue_get(to_main); return file_buffer; } static struct file_info *write_file_empty(struct dir_ent *dir_ent, struct file_buffer *file_buffer, int *duplicate_file) { file_count ++; *duplicate_file = FALSE; cache_block_put(file_buffer); return create_non_dup(0, 0, 0, 0, NULL, 0, &empty_fragment, 0, 0, FALSE, FALSE); } static struct file_info *write_file_frag(struct dir_ent *dir_ent, struct file_buffer *file_buffer, int *duplicate_file) { int size = file_buffer->file_size; struct fragment *fragment; unsigned short checksum = file_buffer->checksum; struct file_info *file; file = frag_duplicate(file_buffer, duplicate_file); if(!file) { fragment = get_and_fill_fragment(file_buffer, dir_ent, FALSE); if(duplicate_checking) file = add_non_dup(size, 0, 0, 0, NULL, 0, fragment, 0, checksum, TRUE, TRUE, FALSE, FALSE, 0); else file = create_non_dup(size, 0, 0, 0, NULL, 0, fragment, 0, checksum, TRUE, TRUE); } cache_block_put(file_buffer); total_bytes += size; file_count ++; inc_progress_bar(); return file; } static void log_file(struct dir_ent *dir_ent, long long start) { if(logging && start) fprintf(log_fd, "%s, %lld\n", pathname(dir_ent), start); } static struct file_info *write_file_process(int *status, struct dir_ent *dir_ent, struct file_buffer *read_buffer, int *duplicate_file) { long long read_size, file_bytes, start; struct fragment *fragment; unsigned int *block_list = NULL; int block = 0; long long sparse = 0; struct file_buffer *fragment_buffer = NULL; struct file_info *file; *duplicate_file = FALSE; if(reproducible) ensure_fragments_flushed(); else lock_fragments(); file_bytes = 0; start = bytes; while (1) { read_size = read_buffer->file_size; if(read_buffer->fragment) { fragment_buffer = read_buffer; if(block == 0) start=0; } else { block_list = realloc(block_list, (block + 1) * sizeof(unsigned int)); if(block_list == NULL) MEM_ERROR(); block_list[block ++] = read_buffer->c_byte; if(read_buffer->c_byte) { read_buffer->block = bytes; bytes += read_buffer->size; cache_hash(read_buffer, read_buffer->block); file_bytes += read_buffer->size; queue_put(to_writer, read_buffer); } else { sparse += read_buffer->size; cache_block_put(read_buffer); } } inc_progress_bar(); if(read_size != -1) break; read_buffer = get_file_buffer(); if(read_buffer->error) goto read_err; } if(!reproducible) unlock_fragments(); fragment = get_and_fill_fragment(fragment_buffer, dir_ent, block != 0); if(duplicate_checking) { int bl_hash = block ? block_hash(block_list[0], block) : 0; file = add_non_dup(read_size, file_bytes, block, sparse, block_list, start, fragment, 0, fragment_buffer ? fragment_buffer->checksum : 0, FALSE, TRUE, FALSE, FALSE, bl_hash); } else file = create_non_dup(read_size, file_bytes, block, sparse, block_list, start, fragment, 0, fragment_buffer ? fragment_buffer->checksum : 0, FALSE, TRUE); cache_block_put(fragment_buffer); file_count ++; total_bytes += read_size; log_file(dir_ent, start); *status = 0; return file; read_err: dec_progress_bar(block); *status = read_buffer->error; bytes = start; if(!block_device) { int res; queue_put(to_writer, NULL); if(queue_get(from_writer) != 0) EXIT_MKSQUASHFS(); res = ftruncate(fd, bytes); if(res != 0) BAD_ERROR("Failed to truncate dest file because %s\n", strerror(errno)); } if(!reproducible) unlock_fragments(); free(block_list); cache_block_put(read_buffer); return NULL; } static struct file_info *write_file_blocks_dup(int *status, struct dir_ent *dir_ent, struct file_buffer *read_buffer, int *duplicate_file, int bl_hash) { int block, thresh; long long read_size = read_buffer->file_size; long long file_bytes, start; int blocks = (read_size + block_size - 1) >> block_log; unsigned int *block_list; struct file_buffer **buffer_list; long long sparse = 0; struct file_buffer *fragment_buffer = NULL; struct file_info *file; int block_dup; block_list = malloc(blocks * sizeof(unsigned int)); if(block_list == NULL) MEM_ERROR(); buffer_list = malloc(blocks * sizeof(struct file_buffer *)); if(buffer_list == NULL) MEM_ERROR(); if(reproducible) ensure_fragments_flushed(); else lock_fragments(); file_bytes = 0; start = bytes; thresh = blocks > bwriter_size ? blocks - bwriter_size : 0; for(block = 0; block < blocks;) { if(read_buffer->fragment) { block_list[block] = 0; buffer_list[block] = NULL; fragment_buffer = read_buffer; blocks = read_size >> block_log; } else { block_list[block] = read_buffer->c_byte; if(read_buffer->c_byte) { read_buffer->block = bytes; bytes += read_buffer->size; file_bytes += read_buffer->size; cache_hash(read_buffer, read_buffer->block); if(block < thresh) { buffer_list[block] = NULL; queue_put(to_writer, read_buffer); } else buffer_list[block] = read_buffer; } else { buffer_list[block] = NULL; sparse += read_buffer->size; cache_block_put(read_buffer); } } inc_progress_bar(); if(++block < blocks) { read_buffer = get_file_buffer(); if(read_buffer->error) goto read_err; } } /* * sparse count is needed to ensure squashfs correctly reports a * a smaller block count on stat calls to sparse files. This is * to ensure intelligent applications like cp correctly handle the * file as a sparse file. If the file in the original filesystem isn't * stored as a sparse file then still store it sparsely in squashfs, but * report it as non-sparse on stat calls to preserve semantics */ if(sparse && (dir_ent->inode->buf.st_blocks << 9) >= read_size) sparse = 0; file = duplicate(duplicate_file, &block_dup, read_size, file_bytes, block_list, start, dir_ent, fragment_buffer, blocks, sparse, bl_hash); if(block_dup == FALSE) { for(block = thresh; block < blocks; block ++) if(buffer_list[block]) queue_put(to_writer, buffer_list[block]); } else { for(block = thresh; block < blocks; block ++) cache_block_put(buffer_list[block]); bytes = start; if(thresh && !block_device) { int res; queue_put(to_writer, NULL); if(queue_get(from_writer) != 0) EXIT_MKSQUASHFS(); res = ftruncate(fd, bytes); if(res != 0) BAD_ERROR("Failed to truncate dest file because" " %s\n", strerror(errno)); } } if(!reproducible) unlock_fragments(); cache_block_put(fragment_buffer); free(buffer_list); file_count ++; total_bytes += read_size; if(block_dup == TRUE) free(block_list); else log_file(dir_ent, file->start); *status = 0; return file; read_err: dec_progress_bar(block); *status = read_buffer->error; bytes = start; if(thresh && !block_device) { int res; queue_put(to_writer, NULL); if(queue_get(from_writer) != 0) EXIT_MKSQUASHFS(); res = ftruncate(fd, bytes); if(res != 0) BAD_ERROR("Failed to truncate dest file because %s\n", strerror(errno)); } if(!reproducible) unlock_fragments(); for(blocks = thresh; blocks < block; blocks ++) cache_block_put(buffer_list[blocks]); free(buffer_list); free(block_list); cache_block_put(read_buffer); return NULL; } static struct file_info *write_file_blocks(int *status, struct dir_ent *dir_ent, struct file_buffer *read_buffer, int *dup) { long long read_size = read_buffer->file_size; long long file_bytes, start; struct fragment *fragment; unsigned int *block_list; int block; int blocks = (read_size + block_size - 1) >> block_log; long long sparse = 0; struct file_buffer *fragment_buffer = NULL; struct file_info *file; int bl_hash = 0; if(pre_duplicate(read_size, dir_ent->inode, read_buffer, &bl_hash)) return write_file_blocks_dup(status, dir_ent, read_buffer, dup, bl_hash); *dup = FALSE; block_list = malloc(blocks * sizeof(unsigned int)); if(block_list == NULL) MEM_ERROR(); if(reproducible) ensure_fragments_flushed(); else lock_fragments(); file_bytes = 0; start = bytes; for(block = 0; block < blocks;) { if(read_buffer->fragment) { block_list[block] = 0; fragment_buffer = read_buffer; blocks = read_size >> block_log; } else { block_list[block] = read_buffer->c_byte; if(read_buffer->c_byte) { read_buffer->block = bytes; bytes += read_buffer->size; cache_hash(read_buffer, read_buffer->block); file_bytes += read_buffer->size; queue_put(to_writer, read_buffer); } else { sparse += read_buffer->size; cache_block_put(read_buffer); } } inc_progress_bar(); if(++block < blocks) { read_buffer = get_file_buffer(); if(read_buffer->error) goto read_err; } } /* * sparse count is needed to ensure squashfs correctly reports a * a smaller block count on stat calls to sparse files. This is * to ensure intelligent applications like cp correctly handle the * file as a sparse file. If the file in the original filesystem isn't * stored as a sparse file then still store it sparsely in squashfs, but * report it as non-sparse on stat calls to preserve semantics */ if(sparse && (dir_ent->inode->buf.st_blocks << 9) >= read_size) sparse = 0; if(!reproducible) unlock_fragments(); fragment = get_and_fill_fragment(fragment_buffer, dir_ent, TRUE); if(duplicate_checking) file = add_non_dup(read_size, file_bytes, blocks, sparse, block_list, start, fragment, 0, fragment_buffer ? fragment_buffer->checksum : 0, FALSE, TRUE, FALSE, FALSE, bl_hash); else file = create_non_dup(read_size, file_bytes, blocks, sparse, block_list, start, fragment, 0, fragment_buffer ? fragment_buffer->checksum : 0, FALSE, TRUE); cache_block_put(fragment_buffer); file_count ++; total_bytes += read_size; log_file(dir_ent, start); *status = 0; return file; read_err: dec_progress_bar(block); *status = read_buffer->error; bytes = start; if(!block_device) { int res; queue_put(to_writer, NULL); if(queue_get(from_writer) != 0) EXIT_MKSQUASHFS(); res = ftruncate(fd, bytes); if(res != 0) BAD_ERROR("Failed to truncate dest file because %s\n", strerror(errno)); } if(!reproducible) unlock_fragments(); free(block_list); cache_block_put(read_buffer); return NULL; } struct file_info *write_file(struct dir_ent *dir, int *dup) { int status; struct file_buffer *read_buffer; struct file_info *file; again: read_buffer = get_file_buffer(); status = read_buffer->error; if(status) cache_block_put(read_buffer); else if(read_buffer->file_size == -1) file = write_file_process(&status, dir, read_buffer, dup); else if(read_buffer->file_size == 0) file = write_file_empty(dir, read_buffer, dup); else if(read_buffer->fragment && read_buffer->c_byte) file = write_file_frag(dir, read_buffer, dup); else file = write_file_blocks(&status, dir, read_buffer, dup); if(status == 2) { ERROR("File %s changed size while reading filesystem, " "attempting to re-read\n", pathname(dir)); goto again; } else if(status == 1) { ERROR_START("Failed to read file %s", pathname(dir)); ERROR_EXIT(", creating empty file\n"); file = write_file_empty(dir, NULL, dup); } else if(status) BAD_ERROR("Unexpected status value in write_file()"); return file; } #define BUFF_SIZE 512 char *name; static char *basename_r(); static char *getbase(char *pathname) { static char *b_buffer = NULL; static int b_size = BUFF_SIZE; char *result; if(b_buffer == NULL) { b_buffer = malloc(b_size); if(b_buffer == NULL) MEM_ERROR(); } while(1) { if(*pathname != '/') { result = getcwd(b_buffer, b_size); if(result == NULL && errno != ERANGE) BAD_ERROR("Getcwd failed in getbase\n"); /* enough room for pathname + "/" + '\0' terminator? */ if(result && strlen(pathname) + 2 <= b_size - strlen(b_buffer)) { strcat(strcat(b_buffer, "/"), pathname); break; } } else if(strlen(pathname) < b_size) { strcpy(b_buffer, pathname); break; } /* Buffer not large enough, realloc and try again */ b_buffer = realloc(b_buffer, b_size += BUFF_SIZE); if(b_buffer == NULL) MEM_ERROR(); } name = b_buffer; if(((result = basename_r()) == NULL) || (strcmp(result, "..") == 0)) return NULL; else return result; } static char *basename_r() { char *s; char *p; int n = 1; for(;;) { s = name; if(*name == '\0') return NULL; if(*name != '/') { while(*name != '\0' && *name != '/') name++; n = name - s; } while(*name == '/') name++; if(strncmp(s, ".", n) == 0) continue; if((*name == '\0') || (strncmp(s, "..", n) == 0) || ((p = basename_r()) == NULL)) { s[n] = '\0'; return s; } if(strcmp(p, "..") == 0) continue; return p; } } static inline void dec_nlink_inode(struct dir_ent *dir_ent) { if(dir_ent->inode == NULL || dir_ent->inode->root_entry) return; if(dir_ent->inode->nlink == 1) { /* Delete this inode, as the last or only reference * to it is going away */ struct stat *buf = &dir_ent->inode->buf; int ino_hash = INODE_HASH(buf->st_dev, buf->st_ino); struct inode_info *inode = inode_info[ino_hash]; struct inode_info *prev = NULL; while(inode && inode != dir_ent->inode) { prev = inode; inode = inode->next; } if(inode) { if(prev) prev->next = inode->next; else inode_info[ino_hash] = inode->next; } /* Decrement the progress bar */ if((buf->st_mode & S_IFMT) == S_IFREG) progress_bar_size(-((buf->st_size + block_size - 1) >> block_log)); free(dir_ent->inode); dir_ent->inode = NULL; } else dir_ent->inode->nlink --; } static struct inode_info *lookup_inode3(struct stat *buf, struct pseudo_dev *pseudo, char *symlink, int bytes) { int ino_hash = INODE_HASH(buf->st_dev, buf->st_ino); struct inode_info *inode; /* * Look-up inode in hash table, if it already exists we have a * hardlink, so increment the nlink count and return it. * Don't do the look-up for directories because Unix/Linux doesn't * allow hard-links to directories. */ if ((buf->st_mode & S_IFMT) != S_IFDIR && !no_hardlinks) { for(inode = inode_info[ino_hash]; inode; inode = inode->next) { if(memcmp(buf, &inode->buf, sizeof(struct stat)) == 0) { inode->nlink ++; return inode; } } } if((buf->st_mode & S_IFMT) == S_IFREG) progress_bar_size((buf->st_size + block_size - 1) >> block_log); inode = malloc(sizeof(struct inode_info) + bytes); if(inode == NULL) MEM_ERROR(); if(bytes) memcpy(&inode->symlink, symlink, bytes); memcpy(&inode->buf, buf, sizeof(struct stat)); inode->read = FALSE; inode->root_entry = FALSE; inode->pseudo = pseudo; inode->inode = SQUASHFS_INVALID_BLK; inode->nlink = 1; inode->inode_number = 0; inode->dummy_root_dir = FALSE; inode->xattr = NULL; inode->tarfile = FALSE; /* * Copy filesystem wide defaults into inode, these filesystem * wide defaults may be altered on an individual inode basis by * user specified actions * */ inode->no_fragments = no_fragments; inode->always_use_fragments = always_use_fragments; inode->noD = noD; inode->noF = noF; inode->next = inode_info[ino_hash]; inode_info[ino_hash] = inode; return inode; } static struct inode_info *lookup_inode2(struct stat *buf, struct pseudo_dev *pseudo) { return lookup_inode3(buf, pseudo, NULL, 0); } struct inode_info *lookup_inode(struct stat *buf) { return lookup_inode2(buf, NULL); } static inline void alloc_inode_no(struct inode_info *inode, unsigned int use_this) { if (inode->inode_number == 0) { inode->inode_number = use_this ? : inode_no ++; } } struct dir_info *create_dir(char *pathname, char *subpath, unsigned int depth) { struct dir_info *dir; dir = malloc(sizeof(struct dir_info)); if(dir == NULL) MEM_ERROR(); dir->pathname = strdup(pathname); dir->subpath = strdup(subpath); dir->count = 0; dir->directory_count = 0; dir->dir_is_ldir = TRUE; dir->list = NULL; dir->depth = depth; dir->excluded = 0; return dir; } struct dir_ent *lookup_name(struct dir_info *dir, char *name) { struct dir_ent *dir_ent = dir->list; for(; dir_ent && strcmp(dir_ent->name, name) != 0; dir_ent = dir_ent->next); return dir_ent; } struct dir_ent *create_dir_entry(char *name, char *source_name, char *nonstandard_pathname, struct dir_info *dir) { struct dir_ent *dir_ent = malloc(sizeof(struct dir_ent)); if(dir_ent == NULL) MEM_ERROR(); dir_ent->name = name; dir_ent->source_name = source_name; dir_ent->nonstandard_pathname = nonstandard_pathname; dir_ent->our_dir = dir; dir_ent->inode = NULL; dir_ent->next = NULL; return dir_ent; } void add_dir_entry(struct dir_ent *dir_ent, struct dir_info *sub_dir, struct inode_info *inode_info) { struct dir_info *dir = dir_ent->our_dir; if(sub_dir) sub_dir->dir_ent = dir_ent; dir_ent->inode = inode_info; dir_ent->dir = sub_dir; dir_ent->next = dir->list; dir->list = dir_ent; dir->count++; } static inline void add_dir_entry2(char *name, char *source_name, char *nonstandard_pathname, struct dir_info *sub_dir, struct inode_info *inode_info, struct dir_info *dir) { struct dir_ent *dir_ent = create_dir_entry(name, source_name, nonstandard_pathname, dir); add_dir_entry(dir_ent, sub_dir, inode_info); } void free_dir_entry(struct dir_ent *dir_ent) { if(dir_ent->name) free(dir_ent->name); if(dir_ent->source_name) free(dir_ent->source_name); if(dir_ent->nonstandard_pathname) free(dir_ent->nonstandard_pathname); /* if this entry has been associated with an inode, then we need * to update the inode nlink count */ dec_nlink_inode(dir_ent); free(dir_ent); } static inline void add_excluded(struct dir_info *dir) { dir->excluded ++; } squashfs_inode do_directory_scans(struct dir_ent *dir_ent, int progress) { squashfs_inode inode; struct pseudo *pseudo = get_pseudo(); /* * Process most actions and any pseudo files */ /* if there's a root pseudo definition skip it, it will have already * been handled if no sources specified on command line. * If sources have been specified, then just ignore it, as sources * on the command line take precedence. */ if(pseudo != NULL && pseudo->names == 1 && strcmp(pseudo->name[0].name, "/") == 0) { if(pseudo->name[0].xattr) root_dir->dir_ent->inode->xattr = pseudo->name[0].xattr; pseudo = pseudo->name[0].pseudo; } if(actions() || pseudo) dir_scan2(root_dir, pseudo); /* * Process move actions */ if(move_actions()) { dir_scan3(root_dir); do_move_actions(); } /* * Process prune actions */ if(prune_actions()) { dir_scan4(root_dir, TRUE); dir_scan4(root_dir, FALSE); } /* * Process empty actions */ if(empty_actions()) dir_scan5(root_dir); /* * Sort directories and compute the inode numbers */ dir_scan6(root_dir); alloc_inode_no(dir_ent->inode, root_inode_number); eval_actions(root_dir, dir_ent); if(sorted) generate_file_priorities(root_dir, 0, &root_dir->dir_ent->inode->buf); if(appending) { sigset_t sigmask; restore_thread = init_restore_thread(); sigemptyset(&sigmask); sigaddset(&sigmask, SIGINT); sigaddset(&sigmask, SIGTERM); sigaddset(&sigmask, SIGUSR1); if(pthread_sigmask(SIG_BLOCK, &sigmask, NULL) != 0) BAD_ERROR("Failed to set signal mask\n"); write_destination(fd, SQUASHFS_START, 4, "\0\0\0\0"); } queue_put(to_reader, root_dir); if(sorted) sort_files_and_write(root_dir); dir_scan7(&inode, root_dir); dir_ent->inode->inode = inode; dir_ent->inode->type = SQUASHFS_DIR_TYPE; return inode; } static squashfs_inode scan_single(char *pathname, int progress) { struct stat buf; struct dir_ent *dir_ent; if(appending) root_dir = dir_scan1(pathname, "", paths, scan1_single_readdir, 1); else root_dir = dir_scan1(pathname, "", paths, scan1_readdir, 1); if(root_dir == NULL) BAD_ERROR("Failed to scan source directory\n"); /* Create root directory dir_ent and associated inode, and connect * it to the root directory dir_info structure */ dir_ent = create_dir_entry("", NULL, pathname, scan1_opendir("", "", 0)); if(lstat(pathname, &buf) == -1) /* source directory has disappeared? */ BAD_ERROR("Cannot stat source directory %s because %s\n", pathname, strerror(errno)); if(root_mode_opt) buf.st_mode = root_mode | S_IFDIR; if(root_uid_opt) buf.st_uid = root_uid; if(root_gid_opt) buf.st_gid = root_gid; if(root_time_opt) buf.st_mtime = root_time; if(pseudo_override && global_uid_opt) buf.st_uid = global_uid; if(pseudo_override && global_gid_opt) buf.st_gid = global_gid; dir_ent->inode = lookup_inode(&buf); dir_ent->dir = root_dir; root_dir->dir_ent = dir_ent; return do_directory_scans(dir_ent, progress); } static squashfs_inode scan_encomp(int progress) { struct stat buf; struct dir_ent *dir_ent; root_dir = dir_scan1("", "", paths, scan1_encomp_readdir, 1); if(root_dir == NULL) BAD_ERROR("Failed to scan source\n"); /* Create root directory dir_ent and associated inode, and connect * it to the root directory dir_info structure */ dir_ent = create_dir_entry("", NULL, "", scan1_opendir("", "", 0)); /* * dummy top level directory, multiple sources specified on * command line */ memset(&buf, 0, sizeof(buf)); if(root_mode_opt) buf.st_mode = root_mode | S_IFDIR; else buf.st_mode = S_IRWXU | S_IRWXG | S_IRWXO | S_IFDIR; if(root_uid_opt) buf.st_uid = root_uid; else buf.st_uid = getuid(); if(root_gid_opt) buf.st_gid = root_gid; else buf.st_gid = getgid(); if(root_time_opt) buf.st_mtime = root_time; else buf.st_mtime = time(NULL); if(pseudo_override && global_uid_opt) buf.st_uid = global_uid; if(pseudo_override && global_gid_opt) buf.st_gid = global_gid; buf.st_dev = 0; buf.st_ino = 0; dir_ent->inode = lookup_inode(&buf); dir_ent->inode->dummy_root_dir = TRUE; dir_ent->dir = root_dir; root_dir->dir_ent = dir_ent; return do_directory_scans(dir_ent, progress); } squashfs_inode dir_scan(int directory, int progress) { int single = !keep_as_directory && source == 1; if(single && directory) return scan_single(source_path[0], progress); else return scan_encomp(progress); } /* * dir_scan1 routines... * These scan the source directories into memory for processing. * Exclude actions are processed here (in contrast to the other actions) * because they affect what is scanned. */ struct dir_info *scan1_opendir(char *pathname, char *subpath, unsigned int depth) { struct dir_info *dir; dir = malloc(sizeof(struct dir_info)); if(dir == NULL) MEM_ERROR(); if(pathname[0] != '\0') { dir->linuxdir = opendir(pathname); if(dir->linuxdir == NULL) { free(dir); return NULL; } } dir->pathname = strdup(pathname); dir->subpath = strdup(subpath); dir->count = 0; dir->directory_count = 0; dir->dir_is_ldir = TRUE; dir->list = NULL; dir->depth = depth; dir->excluded = 0; return dir; } static struct dir_ent *scan1_encomp_readdir(struct dir_info *dir) { static int index = 0; if(dir->count < old_root_entries) { int i; for(i = 0; i < old_root_entries; i++) { if(old_root_entry[i].inode.type == SQUASHFS_DIR_TYPE) dir->directory_count ++; add_dir_entry2(old_root_entry[i].name, NULL, NULL, NULL, &old_root_entry[i].inode, dir); } } while(index < source) { char *basename = NULL; char *dir_name = getbase(source_path[index]); int pass = 1, res; if(dir_name == NULL) { ERROR_START("Bad source directory %s", source_path[index]); ERROR_EXIT(" - skipping ...\n"); index ++; continue; } dir_name = strdup(dir_name); for(;;) { struct dir_ent *dir_ent = dir->list; for(; dir_ent && strcmp(dir_ent->name, dir_name) != 0; dir_ent = dir_ent->next); if(dir_ent == NULL) break; ERROR("Source directory entry %s already used! - trying" " ", dir_name); if(pass == 1) basename = dir_name; else free(dir_name); res = asprintf(&dir_name, "%s_%d", basename, pass++); if(res == -1) BAD_ERROR("asprintf failed in " "scan1_encomp_readdir\n"); ERROR("%s\n", dir_name); } if(one_file_system && source > 1) cur_dev = source_dev[index]; return create_dir_entry(dir_name, basename, strdup(source_path[index ++]), dir); } return NULL; } static struct dir_ent *scan1_single_readdir(struct dir_info *dir) { struct dirent *d_name; int i; if(dir->count < old_root_entries) { for(i = 0; i < old_root_entries; i++) { if(old_root_entry[i].inode.type == SQUASHFS_DIR_TYPE) dir->directory_count ++; add_dir_entry2(old_root_entry[i].name, NULL, NULL, NULL, &old_root_entry[i].inode, dir); } } if((d_name = readdir(dir->linuxdir)) != NULL) { char *basename = NULL; char *dir_name = strdup(d_name->d_name); int pass = 1, res; for(;;) { struct dir_ent *dir_ent = dir->list; for(; dir_ent && strcmp(dir_ent->name, dir_name) != 0; dir_ent = dir_ent->next); if(dir_ent == NULL) break; ERROR("Source directory entry %s already used! - trying" " ", dir_name); if (pass == 1) basename = dir_name; else free(dir_name); res = asprintf(&dir_name, "%s_%d", d_name->d_name, pass++); if(res == -1) BAD_ERROR("asprintf failed in " "scan1_single_readdir\n"); ERROR("%s\n", dir_name); } return create_dir_entry(dir_name, basename, NULL, dir); } return NULL; } static struct dir_ent *scan1_readdir(struct dir_info *dir) { struct dirent *d_name = readdir(dir->linuxdir); return d_name ? create_dir_entry(strdup(d_name->d_name), NULL, NULL, dir) : NULL; } static void scan1_freedir(struct dir_info *dir) { if(dir->pathname[0] != '\0') closedir(dir->linuxdir); } static struct dir_info *dir_scan1(char *filename, char *subpath, struct pathnames *paths, struct dir_ent *(_readdir)(struct dir_info *), unsigned int depth) { struct dir_info *dir = scan1_opendir(filename, subpath, depth); struct dir_ent *dir_ent; if(dir == NULL) { ERROR_START("Could not open %s", filename); ERROR_EXIT(", skipping...\n"); return NULL; } if(max_depth_opt && depth > max_depth) { add_excluded(dir); scan1_freedir(dir); return dir; } while((dir_ent = _readdir(dir))) { struct dir_info *sub_dir; struct stat buf; struct pathnames *new = NULL; char *filename = pathname(dir_ent); char *subpath = NULL; char *dir_name = dir_ent->name; int create_empty_directory = FALSE; if(strcmp(dir_name, ".") == 0 || strcmp(dir_name, "..") == 0) { free_dir_entry(dir_ent); continue; } if(lstat(filename, &buf) == -1) { ERROR_START("Cannot stat dir/file %s because %s", filename, strerror(errno)); ERROR_EXIT(", ignoring\n"); free_dir_entry(dir_ent); continue; } if(one_file_system) { if(buf.st_dev != cur_dev) { if(!S_ISDIR(buf.st_mode) || one_file_system_x) { ERROR("%s is on a different filesystem, ignored\n", filename); free_dir_entry(dir_ent); continue; } create_empty_directory = TRUE; } } if((buf.st_mode & S_IFMT) != S_IFREG && (buf.st_mode & S_IFMT) != S_IFDIR && (buf.st_mode & S_IFMT) != S_IFLNK && (buf.st_mode & S_IFMT) != S_IFCHR && (buf.st_mode & S_IFMT) != S_IFBLK && (buf.st_mode & S_IFMT) != S_IFIFO && (buf.st_mode & S_IFMT) != S_IFSOCK) { ERROR_START("File %s has unrecognised filetype %d", filename, buf.st_mode & S_IFMT); ERROR_EXIT(", ignoring\n"); free_dir_entry(dir_ent); continue; } if(old_exclude && old_excluded(filename, &buf)) { add_excluded(dir); free_dir_entry(dir_ent); continue; } if(!old_exclude && excluded(dir_name, paths, &new)) { add_excluded(dir); free_dir_entry(dir_ent); continue; } if(exclude_actions()) { subpath = subpathname(dir_ent); if(eval_exclude_actions(dir_name, filename, subpath, &buf, depth, dir_ent)) { add_excluded(dir); free_dir_entry(dir_ent); continue; } } switch(buf.st_mode & S_IFMT) { case S_IFDIR: if(subpath == NULL) subpath = subpathname(dir_ent); if(create_empty_directory) { ERROR("%s is on a different filesystem, creating empty directory\n", filename); sub_dir = create_dir(filename, subpath, depth + 1); } else sub_dir = dir_scan1(filename, subpath, new, scan1_readdir, depth + 1); if(sub_dir) { dir->directory_count ++; add_dir_entry(dir_ent, sub_dir, lookup_inode(&buf)); } else free_dir_entry(dir_ent); break; case S_IFLNK: { int byte; static char buff[65536]; /* overflow safe */ byte = readlink(filename, buff, 65536); if(byte == -1) { ERROR_START("Failed to read symlink %s", filename); ERROR_EXIT(", ignoring\n"); } else if(byte == 65536) { ERROR_START("Symlink %s is greater than 65536 " "bytes!", filename); ERROR_EXIT(", ignoring\n"); } else { /* readlink doesn't 0 terminate the returned * path */ buff[byte] = '\0'; add_dir_entry(dir_ent, NULL, lookup_inode3(&buf, NULL, buff, byte + 1)); } break; } default: add_dir_entry(dir_ent, NULL, lookup_inode(&buf)); } free(new); } scan1_freedir(dir); return dir; } /* * dir_scan2 routines... * This processes most actions and any pseudo files */ static struct dir_ent *scan2_readdir(struct dir_info *dir, struct dir_ent *dir_ent) { if (dir_ent == NULL) dir_ent = dir->list; else dir_ent = dir_ent->next; for(; dir_ent && dir_ent->inode->root_entry; dir_ent = dir_ent->next); return dir_ent; } static void dir_scan2(struct dir_info *dir, struct pseudo *pseudo) { struct dir_ent *dirent = NULL; struct pseudo_entry *pseudo_ent; struct stat buf; while((dirent = scan2_readdir(dir, dirent)) != NULL) { struct inode_info *inode_info = dirent->inode; struct stat *buf = &inode_info->buf; char *name = dirent->name; eval_actions(root_dir, dirent); if(pseudo_override && global_uid_opt) buf->st_uid = global_uid; if(pseudo_override && global_gid_opt) buf->st_gid = global_gid; if((buf->st_mode & S_IFMT) == S_IFDIR) dir_scan2(dirent->dir, pseudo_subdir(name, pseudo)); } /* * Process pseudo modify and add (file, directory etc) definitions */ while((pseudo_ent = pseudo_readdir(pseudo)) != NULL) { struct dir_ent *dir_ent = NULL; if(appending && dir->depth == 1) { dir_ent = lookup_name(dir, pseudo_ent->name); if(dir_ent && dir_ent->inode->root_entry) { BAD_ERROR("Pseudo files: File \"%s\" " "already exists in root directory of " "the\nfilesystem being appended to. " "Pseudo definitions can\'t modify it " "or (if directory) add files to it\n", pseudo_ent->name); } } if(pseudo_ent->dev == NULL) continue; if(dir_ent == NULL) dir_ent = lookup_name(dir, pseudo_ent->name); if(pseudo_ent->dev->type == 'm' || pseudo_ent->dev->type == 'M') { struct stat *buf; if(dir_ent == NULL) { ERROR_START("Pseudo modify file \"%s\" does " "not exist in source filesystem.", pseudo_ent->pathname); ERROR_EXIT(" Ignoring.\n"); continue; } buf = &dir_ent->inode->buf; buf->st_mode = (buf->st_mode & S_IFMT) | pseudo_ent->dev->buf->mode; buf->st_uid = pseudo_ent->dev->buf->uid; buf->st_gid = pseudo_ent->dev->buf->gid; if(pseudo_ent->dev->type == 'M') buf->st_mtime = pseudo_ent->dev->buf->mtime; continue; } if(dir_ent) { ERROR_START("Pseudo file \"%s\" exists in source " "filesystem \"%s\".", pseudo_ent->pathname, pathname(dir_ent)); ERROR_EXIT("\nIgnoring, exclude it (-e/-ef) to override.\n"); continue; } if(pseudo_ent->dev->type != 'l') { memset(&buf, 0, sizeof(buf)); buf.st_mode = pseudo_ent->dev->buf->mode; buf.st_uid = pseudo_ent->dev->buf->uid; buf.st_gid = pseudo_ent->dev->buf->gid; buf.st_rdev = makedev(pseudo_ent->dev->buf->major, pseudo_ent->dev->buf->minor); buf.st_mtime = pseudo_ent->dev->buf->mtime; buf.st_ino = pseudo_ent->dev->buf->ino; if(pseudo_ent->dev->type == 'r') { buf.st_size = pseudo_ent->dev->data->length; if(pseudo_ent->dev->data->sparse == FALSE) buf.st_blocks = (buf.st_size + 511) >> 9; } } if(pseudo_ent->dev->type == 'd') { struct dir_ent *dir_ent = create_dir_entry(pseudo_ent->name, NULL, pseudo_ent->pathname, dir); char *subpath = subpathname(dir_ent); struct dir_info *sub_dir = scan1_opendir("", subpath, dir->depth + 1); dir_scan2(sub_dir, pseudo_ent->pseudo); dir->directory_count ++; add_dir_entry(dir_ent, sub_dir, lookup_inode2(&buf, pseudo_ent->dev)); } else if(pseudo_ent->dev->type == 's') { add_dir_entry2(pseudo_ent->name, NULL, pseudo_ent->pathname, NULL, lookup_inode3(&buf, pseudo_ent->dev, pseudo_ent->dev->symlink, strlen(pseudo_ent->dev->symlink) + 1), dir); } else if(pseudo_ent->dev->type == 'l') { add_dir_entry2(pseudo_ent->name, NULL, pseudo_ent->dev->linkname, NULL, lookup_inode(pseudo_ent->dev->linkbuf), dir); } else { add_dir_entry2(pseudo_ent->name, NULL, pseudo_ent->pathname, NULL, lookup_inode2(&buf, pseudo_ent->dev), dir); } } /* * Process pseudo xattr definitions */ if(pseudo) pseudo->count = 0; while((pseudo_ent = pseudo_readdir(pseudo)) != NULL) { struct dir_ent *dir_ent = NULL; if(pseudo_ent->xattr == NULL) continue; dir_ent = lookup_name(dir, pseudo_ent->name); if(dir_ent == NULL) { ERROR_START("Pseudo xattr file \"%s\" does not " "exist in source filesystem.", pseudo_ent->pathname); ERROR_EXIT(" Ignoring.\n"); continue; } dir_ent->inode->xattr = pseudo_ent->xattr; } } /* * dir_scan3 routines... * This processes the move action */ static void dir_scan3(struct dir_info *dir) { struct dir_ent *dir_ent = NULL; while((dir_ent = scan2_readdir(dir, dir_ent)) != NULL) { eval_move_actions(root_dir, dir_ent); if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) dir_scan3(dir_ent->dir); } } /* * dir_scan4 routines... * This processes the prune action. This action is designed to do fine * grained tuning of the in-core directory structure after the exclude, * move and pseudo actions have been performed. This allows complex * tests to be performed which are impossible at exclude time (i.e. * tests which rely on the in-core directory structure) */ void free_dir(struct dir_info *dir) { struct dir_ent *dir_ent = dir->list; while(dir_ent) { struct dir_ent *tmp = dir_ent; if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) if(dir_ent->dir) free_dir(dir_ent->dir); dir_ent = dir_ent->next; free_dir_entry(tmp); } free(dir->pathname); free(dir->subpath); free(dir); } static void dir_scan4(struct dir_info *dir, int symlink) { struct dir_ent *dir_ent = dir->list, *prev = NULL; while(dir_ent) { if(dir_ent->inode->root_entry) { prev = dir_ent; dir_ent = dir_ent->next; continue; } if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) dir_scan4(dir_ent->dir, symlink); if(symlink != ((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFLNK)) { prev = dir_ent; dir_ent = dir_ent->next; continue; } if(eval_prune_actions(root_dir, dir_ent)) { struct dir_ent *tmp = dir_ent; if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) { free_dir(dir_ent->dir); dir->directory_count --; } dir->count --; /* remove dir_ent from list */ dir_ent = dir_ent->next; if(prev) prev->next = dir_ent; else dir->list = dir_ent; /* free it */ free_dir_entry(tmp); add_excluded(dir); continue; } prev = dir_ent; dir_ent = dir_ent->next; } } /* * dir_scan5 routines... * This processes the empty action. This action has to be processed after * all other actions because the previous exclude and move actions and the * pseudo actions affect whether a directory is empty */ static void dir_scan5(struct dir_info *dir) { struct dir_ent *dir_ent = dir->list, *prev = NULL; while(dir_ent) { if(dir_ent->inode->root_entry) { prev = dir_ent; dir_ent = dir_ent->next; continue; } if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) { dir_scan5(dir_ent->dir); if(eval_empty_actions(root_dir, dir_ent)) { struct dir_ent *tmp = dir_ent; /* * delete sub-directory, this is by definition * empty */ free(dir_ent->dir->pathname); free(dir_ent->dir->subpath); free(dir_ent->dir); /* remove dir_ent from list */ dir_ent = dir_ent->next; if(prev) prev->next = dir_ent; else dir->list = dir_ent; /* free it */ free_dir_entry(tmp); /* update counts */ dir->directory_count --; dir->count --; add_excluded(dir); continue; } } prev = dir_ent; dir_ent = dir_ent->next; } } /* * dir_scan6 routines... * This sorts every directory and computes the inode numbers */ /* * Instantiate bottom up linked list merge sort. * * Qsort and other O(n log n) algorithms work well with arrays but not * linked lists. Merge sort another O(n log n) sort algorithm on the other hand * is not ideal for arrays (as it needs an additonal n storage locations * as sorting is not done in place), but it is ideal for linked lists because * it doesn't require any extra storage, */ SORT(sort_directory, dir_ent, name, next); static void dir_scan6(struct dir_info *dir) { struct dir_ent *dir_ent; unsigned int byte_count = 0; sort_directory(&(dir->list), dir->count); for(dir_ent = dir->list; dir_ent; dir_ent = dir_ent->next) { byte_count += strlen(dir_ent->name) + sizeof(struct squashfs_dir_entry); if(dir_ent->inode->root_entry) continue; alloc_inode_no(dir_ent->inode, 0); if((dir_ent->inode->buf.st_mode & S_IFMT) == S_IFDIR) dir_scan6(dir_ent->dir); } if((dir->count < 257 && byte_count < SQUASHFS_METADATA_SIZE)) dir->dir_is_ldir = FALSE; } /* * dir_scan6 routines... * This generates the filesystem metadata and writes it out to the destination */ static void scan7_init_dir(struct directory *dir) { dir->buff = malloc(SQUASHFS_METADATA_SIZE); if(dir->buff == NULL) MEM_ERROR(); dir->size = SQUASHFS_METADATA_SIZE; dir->p = dir->index_count_p = dir->buff; dir->entry_count = 256; dir->entry_count_p = NULL; dir->index = NULL; dir->i_count = dir->i_size = 0; } static struct dir_ent *scan7_readdir(struct directory *dir, struct dir_info *dir_info, struct dir_ent *dir_ent) { if (dir_ent == NULL) dir_ent = dir_info->list; else dir_ent = dir_ent->next; for(; dir_ent && dir_ent->inode->root_entry; dir_ent = dir_ent->next) add_dir(dir_ent->inode->inode, dir_ent->inode->inode_number, dir_ent->name, dir_ent->inode->type, dir); return dir_ent; } static void scan7_freedir(struct directory *dir) { if(dir->index) free(dir->index); free(dir->buff); } static void dir_scan7(squashfs_inode *inode, struct dir_info *dir_info) { int squashfs_type; int duplicate_file; struct directory dir; struct dir_ent *dir_ent = NULL; struct file_info *file; scan7_init_dir(&dir); while((dir_ent = scan7_readdir(&dir, dir_info, dir_ent)) != NULL) { struct stat *buf = &dir_ent->inode->buf; update_info(dir_ent); if(dir_ent->inode->inode == SQUASHFS_INVALID_BLK) { switch(buf->st_mode & S_IFMT) { case S_IFREG: if(dir_ent->inode->tarfile && dir_ent->inode->tar_file->file) file = dir_ent->inode->tar_file->file; else { file = write_file(dir_ent, &duplicate_file); INFO("file %s, uncompressed size %lld " "bytes %s\n", subpathname(dir_ent), (long long) buf->st_size, duplicate_file ? "DUPLICATE" : ""); } squashfs_type = SQUASHFS_FILE_TYPE; *inode = create_inode(NULL, dir_ent, squashfs_type, file->file_size, file->start, file->blocks, file->block_list, file->fragment, NULL, file->sparse); if((duplicate_checking == FALSE && !(tarfile && no_hardlinks)) || file->file_size == 0) { free_fragment(file->fragment); free(file->block_list); free(file); } break; case S_IFDIR: squashfs_type = SQUASHFS_DIR_TYPE; dir_scan7(inode, dir_ent->dir); break; case S_IFLNK: squashfs_type = SQUASHFS_SYMLINK_TYPE; *inode = create_inode(NULL, dir_ent, squashfs_type, 0, 0, 0, NULL, NULL, NULL, 0); INFO("symbolic link %s inode 0x%llx\n", subpathname(dir_ent), *inode); sym_count ++; break; case S_IFCHR: squashfs_type = SQUASHFS_CHRDEV_TYPE; *inode = create_inode(NULL, dir_ent, squashfs_type, 0, 0, 0, NULL, NULL, NULL, 0); INFO("character device %s inode 0x%llx" "\n", subpathname(dir_ent), *inode); dev_count ++; break; case S_IFBLK: squashfs_type = SQUASHFS_BLKDEV_TYPE; *inode = create_inode(NULL, dir_ent, squashfs_type, 0, 0, 0, NULL, NULL, NULL, 0); INFO("block device %s inode 0x%llx\n", subpathname(dir_ent), *inode); dev_count ++; break; case S_IFIFO: squashfs_type = SQUASHFS_FIFO_TYPE; *inode = create_inode(NULL, dir_ent, squashfs_type, 0, 0, 0, NULL, NULL, NULL, 0); INFO("fifo %s inode 0x%llx\n", subpathname(dir_ent), *inode); fifo_count ++; break; case S_IFSOCK: squashfs_type = SQUASHFS_SOCKET_TYPE; *inode = create_inode(NULL, dir_ent, squashfs_type, 0, 0, 0, NULL, NULL, NULL, 0); INFO("unix domain socket %s inode " "0x%llx\n", subpathname(dir_ent), *inode); sock_count ++; break; default: BAD_ERROR("%s unrecognised file type, " "mode is %x\n", subpathname(dir_ent), buf->st_mode); } dir_ent->inode->inode = *inode; dir_ent->inode->type = squashfs_type; } else { *inode = dir_ent->inode->inode; squashfs_type = dir_ent->inode->type; switch(squashfs_type) { case SQUASHFS_FILE_TYPE: if(!sorted) INFO("file %s, uncompressed " "size %lld bytes LINK" "\n", subpathname(dir_ent), (long long) buf->st_size); break; case SQUASHFS_SYMLINK_TYPE: INFO("symbolic link %s inode 0x%llx " "LINK\n", subpathname(dir_ent), *inode); break; case SQUASHFS_CHRDEV_TYPE: INFO("character device %s inode 0x%llx " "LINK\n", subpathname(dir_ent), *inode); break; case SQUASHFS_BLKDEV_TYPE: INFO("block device %s inode 0x%llx " "LINK\n", subpathname(dir_ent), *inode); break; case SQUASHFS_FIFO_TYPE: INFO("fifo %s inode 0x%llx LINK\n", subpathname(dir_ent), *inode); break; case SQUASHFS_SOCKET_TYPE: INFO("unix domain socket %s inode " "0x%llx LINK\n", subpathname(dir_ent), *inode); break; } hardlnk_count ++; } add_dir(*inode, get_inode_no(dir_ent->inode), dir_ent->name, squashfs_type, &dir); } *inode = write_dir(dir_info, &dir); INFO("directory %s inode 0x%llx\n", subpathname(dir_info->dir_ent), *inode); scan7_freedir(&dir); } static void handle_root_entries(struct dir_info *dir) { int i; if(dir->count == 0) { for(i = 0; i < old_root_entries; i++) { if(old_root_entry[i].inode.type == SQUASHFS_DIR_TYPE) dir->directory_count ++; add_dir_entry2(strdup(old_root_entry[i].name), NULL, NULL, NULL, &old_root_entry[i].inode, dir); } } } static char *walk_source(char *source, char **pathname, char **name) { char *path = source, *start; while(*source == '/') source ++; start = source; while(*source != '/' && *source != '\0') source ++; *name = strndup(start, source - start); if(*pathname == NULL) *pathname = strndup(path, source - path); else { char *orig = *pathname; int size = strlen(orig) + (source - path) + 2; *pathname = malloc(size); strcpy(*pathname, orig); strcat(*pathname, "/"); strncat(*pathname, path, source - path); } while(*source == '/') source ++; return source; } static struct dir_info *add_source(struct dir_info *sdir, char *source, char *subpath, char *file, char **prefix, struct pathnames *paths, unsigned int depth) { struct dir_info *sub; struct dir_ent *entry; struct pathnames *new = NULL; struct dir_info *dir = sdir; struct stat buf; char *name, *newsubpath = NULL; int res; if(max_depth_opt && depth > max_depth) return NULL; if(dir == NULL) dir = create_dir("", subpath, depth); if(depth == 1) *prefix = source[0] == '/' ? strdup("/") : strdup("."); if(appending && file == NULL) handle_root_entries(dir); source = walk_source(source, &file, &name); while(depth == 1 && (name[0] == '\0' || strcmp(name, "..") == 0 || strcmp(name, ".") == 0)){ char *old = file; if(name[0] == '\0' || source[0] == '\0') { /* Ran out of pathname skipping leading ".." and "." * If cpiostyle, just ignore it, find always produces * these if run as "find ." or "find .." etc. * * If tarstyle after skipping what we *must* skip * in the pathname (we can't store directories named * ".." or "." or simply "/") there's nothing left after * stripping (i.e. someone just typed "..", "." on * the command line). This isn't what -tarstyle is * intended for, and Mksquashfs without -tarstyle * can handle this scenario */ if(cpiostyle) goto failed_early; else BAD_ERROR("Empty source after stripping '/', " "'..' and '.'. Run Mksquashfs without " "-tarstyle to handle this!\n"); } source = walk_source(source, &file, &name); if(name[0] == '\0' || strcmp(name, "..") == 0 || strcmp(name, ".") == 0) free(old); else *prefix = old; } if((strcmp(name, ".") == 0) || strcmp(name, "..") == 0) BAD_ERROR("Source path can't have '.' or '..' embedded in it with -tarstyle/-cpiostyle[0]\n"); res = lstat(file, &buf); if (res == -1) BAD_ERROR("Can't stat %s because %s\n", file, strerror(errno)); entry = lookup_name(dir, name); if(entry) { /* * name already there. This must be the same file, otherwise * we have a clash, as we can't have two different files with * the same pathname. * * An original root entry from the file being appended to * is never the same file. */ if(entry->inode->root_entry) BAD_ERROR("Source %s conflicts with name in filesystem " "being appended to\n", name); res = memcmp(&buf, &(entry->inode->buf), sizeof(buf)); if(res) BAD_ERROR("Can't have two different sources with same " "pathname\n"); /* * Matching file. * * For tarstyle source handling (leaf directores are * recursively descended) * * - If we're at the leaf of the source, then we either match * or encompass this pre-existing include. So delete any * sub-directories of this pre-existing include. * * - If we're not at the leaf of the source, but we're at * the leaf of the pre-existing include, then the * pre-existing include encompasses this source. So nothing * more to do. * * - Otherwise this is not the leaf of the source, or the leaf * of the pre-existing include, so recurse continuing walking * the source. * * For cpiostyle source handling (leaf directories are not * recursively descended) * * - If we're at the leaf of the source, then we have a * pre-existing include. So nothing to do. * * - If we're not at the leaf of the source, but we're at * the leaf of the pre-existing include, then recurse * walking the source. * * - Otherwise this is not the leaf of the source, or the leaf * of the pre-existing include, so recurse continuing walking * the source. */ if(source[0] == '\0') { if(tarstyle && entry->dir) { free_dir(entry->dir); entry->dir = NULL; } } else if(S_ISDIR(buf.st_mode)) { if(cpiostyle || entry->dir) { excluded(entry->name, paths, &new); subpath = subpathname(entry); sub = add_source(entry->dir, source, subpath, file, prefix, new, depth + 1); if(sub == NULL) goto failed_match; entry->dir = sub; sub->dir_ent = entry; } } else BAD_ERROR("Source component %s is not a directory\n", name); free(name); free(file); } else { /* * No matching name found. * * - If we're at the leaf of the source, then add it. * * - If we're not at the leaf of the source, we will add it, * and recurse walking the source */ if(old_exclude && old_excluded(file, &buf)) goto failed_early; if(old_exclude == FALSE && excluded(name, paths, &new)) goto failed_early; entry = create_dir_entry(name, NULL, file, dir); if(exclude_actions()) { newsubpath = subpathname(entry); if(eval_exclude_actions(name, file, newsubpath, &buf, depth, entry)) { goto failed_entry; } } if(source[0] == '\0' && S_ISLNK(buf.st_mode)) { int byte; static char buff[65536]; /* overflow safe */ struct inode_info *i; byte = readlink(file, buff, 65536); if(byte == -1) BAD_ERROR("Failed to read source symlink %s", file); else if(byte == 65536) BAD_ERROR("Symlink %s is greater than 65536 " "bytes!", file); /* readlink doesn't 0 terminate the returned path */ buff[byte] = '\0'; i = lookup_inode3(&buf, NULL, buff, byte + 1); add_dir_entry(entry, NULL, i); } else if(source[0] == '\0') { add_dir_entry(entry, NULL, lookup_inode(&buf)); if(S_ISDIR(buf.st_mode)) dir->directory_count ++; } else if(S_ISDIR(buf.st_mode)) { if(newsubpath == NULL) newsubpath = subpathname(entry); sub = add_source(NULL, source, newsubpath, file, prefix, new, depth + 1); if(sub == NULL) goto failed_entry; add_dir_entry(entry, sub, lookup_inode(&buf)); dir->directory_count ++; } else BAD_ERROR("Source component %s is not a directory\n", name); } free(new); return dir; failed_early: free(new); free(name); free(file); if(sdir == NULL) free_dir(dir); return NULL; failed_entry: free(new); free_dir_entry(entry); if(sdir == NULL) free_dir(dir); return NULL; failed_match: free(new); free(name); free(file); return NULL; } static struct dir_info *populate_tree(struct dir_info *dir, struct pathnames *paths) { struct dir_ent *entry; struct dir_info *new; for(entry = dir->list; entry; entry = entry->next) if(S_ISDIR(entry->inode->buf.st_mode) && !entry->inode->root_entry) { struct pathnames *newp = NULL; excluded(entry->name, paths, &newp); if(entry->dir == NULL && cpiostyle) { entry->dir = create_dir(pathname(entry), subpathname(entry), dir->depth + 1); entry->dir->dir_ent = entry; } else if(entry->dir == NULL) { cur_dev = entry->inode->buf.st_dev; new = dir_scan1(pathname(entry), subpathname(entry), newp, scan1_readdir, dir->depth + 1); if(new == NULL) return NULL; entry->dir = new; new->dir_ent = entry; } else { new = populate_tree(entry->dir, newp); if(new == NULL) return NULL; } free(newp); } return dir; } static char *get_filename_from_stdin(char terminator) { static int path_max = -1; static int bytes = 0; static int size = 0; static char *buffer = NULL; static char *filename = NULL; static char *src = NULL; char *dest = filename; int used = 0; /* Get the maximum pathname size supported on this system */ if(path_max == -1) { #ifdef PATH_MAX path_max = PATH_MAX; #else path_max = pathconf(".", _PC_PATH_MAX); if(path_max <= 0) path_max = 4096; #endif /* limit to no more than 64K */ if(path_max > 65536) path_max = 65536; } if(buffer == NULL) { buffer = malloc(4096); if(buffer == NULL) MEM_ERROR(); } while(1) { if(bytes == 0) { bytes = read_bytes(STDIN_FILENO, buffer, 4096); if(bytes == -1) BAD_ERROR("Failed to read Tar file from STDIN\n"); if(bytes == 0) { if(used) ERROR("Got EOF when reading filename from STDIN, ignoring\n"); free(filename); free(buffer); return NULL; } src = buffer; } if(size - used <= 1) { int offset = dest - filename; char *buff = realloc(filename, size += 100); if(buff == NULL) MEM_ERROR(); dest = buff + offset; filename = buff; } if(*src == terminator) { src++; bytes--; break; } if(used >= (path_max - 1)) BAD_ERROR("Cpiostyle input filename exceeds maximum " "path limit of %d bytes!\n", path_max); *dest++ = *src++; bytes --; used ++; } *dest = '\0'; return filename; } static char *get_next_filename() { static int cur = 0; char *filename; if(cpiostyle) { while(1) { filename = get_filename_from_stdin(filename_terminator); if(filename == NULL || strlen(filename) != 0) break; } return filename; } else if(cur < source) return source_path[cur ++]; else return NULL; } static squashfs_inode process_source(int progress) { int i, res, first = TRUE, same = FALSE; char *filename, *prefix, *pathname; struct stat buf, buf2; struct dir_ent *entry; struct dir_info *new; for(i = 0; (filename = get_next_filename()); i++) { new = add_source(root_dir, filename, "", NULL, &prefix, paths, 1); if(new) { /* does argv[i] start from the same directory? */ if(first) { res = lstat(prefix, &buf); if (res == -1) BAD_ERROR("Can't stat %s because %s\n", prefix, strerror(errno)); first = FALSE; same = TRUE; pathname = strdup(prefix); } else if(same) { res = lstat(prefix, &buf2); if (res == -1) BAD_ERROR("Can't stat %s because %s\n", prefix, strerror(errno)); if(buf.st_dev != buf2.st_dev || buf.st_ino != buf2.st_ino) same = FALSE; } free(prefix); root_dir = new; } } if(root_dir == NULL) { /* Empty directory tree after processing the sources, and * so everything was excluded. * We need to create an empty directory to reflect this, and * if appending, fill it with the original root directory * contents */ root_dir = scan1_opendir("", "", 0); if(appending) handle_root_entries(root_dir); } new = scan1_opendir("", "", 0); if(!same) { /* * Top level directory conflict. Create dummy * top level directory */ memset(&buf, 0, sizeof(buf)); buf.st_mode = (root_mode_opt) ? root_mode | S_IFDIR : S_IRWXU | S_IRWXG | S_IRWXO | S_IFDIR; if(root_uid_opt) buf.st_uid = root_uid; else buf.st_uid = getuid(); if(root_gid_opt) buf.st_gid = root_gid; else buf.st_gid = getgid(); if(root_time_opt) buf.st_mtime = root_time; else buf.st_mtime = time(NULL); if(pseudo_override && global_uid_opt) buf.st_uid = global_uid; if(pseudo_override && global_gid_opt) buf.st_gid = global_gid; entry = create_dir_entry("", NULL, "", new); entry->inode = lookup_inode(&buf); entry->inode->dummy_root_dir = TRUE; } else { if(root_mode_opt) buf.st_mode = root_mode | S_IFDIR; if(root_uid_opt) buf.st_uid = root_uid; if(root_gid_opt) buf.st_gid = root_gid; if(root_time_opt) buf.st_mtime = root_time; if(pseudo_override && global_uid_opt) buf.st_uid = global_uid; if(pseudo_override && global_gid_opt) buf.st_gid = global_gid; entry = create_dir_entry("", NULL, pathname, new); entry->inode = lookup_inode(&buf); } entry->dir = root_dir; root_dir->dir_ent = entry; root_dir = populate_tree(root_dir, paths); if(root_dir == NULL) BAD_ERROR("Failed to read directory hierarchy\n"); return do_directory_scans(entry, progress); } /* * Source directory specified as - which means no source directories * * Here the pseudo definitions will be providing the source directory */ static squashfs_inode no_sources(int progress) { struct stat buf; struct dir_ent *dir_ent; struct pseudo_entry *pseudo_ent; struct pseudo *pseudo = get_pseudo(); if(pseudo == NULL || pseudo->names != 1 || strcmp(pseudo->name[0].name, "/") != 0) { ERROR_START("Source is \"-\", but no pseudo definition for \"/\"\n"); ERROR_EXIT("Did you forget to specify -cpiostyle or -tar?\n"); EXIT_MKSQUASHFS(); } pseudo_ent = &pseudo->name[0]; /* create root directory */ root_dir = scan1_opendir("", "", 1); if(appending) handle_root_entries(root_dir); /* Create root directory dir_ent and associated inode, and connect * it to the root directory dir_info structure */ dir_ent = create_dir_entry("", NULL, "", scan1_opendir("", "", 0)); memset(&buf, 0, sizeof(buf)); if(root_mode_opt) buf.st_mode = root_mode | S_IFDIR; else buf.st_mode = pseudo_ent->dev->buf->mode; if(root_uid_opt) buf.st_uid = root_uid; else buf.st_uid = pseudo_ent->dev->buf->uid; if(root_gid_opt) buf.st_gid = root_gid; else buf.st_gid = pseudo_ent->dev->buf->gid; if(root_time_opt) buf.st_mtime = root_time; else buf.st_mtime = pseudo_ent->dev->buf->mtime; buf.st_ino = pseudo_ent->dev->buf->ino; dir_ent->inode = lookup_inode2(&buf, pseudo_ent->dev); dir_ent->dir = root_dir; root_dir->dir_ent = dir_ent; return do_directory_scans(dir_ent, progress); } static unsigned int slog(unsigned int block) { int i; for(i = 12; i <= 20; i++) if(block == (1 << i)) return i; return 0; } static int old_excluded(char *filename, struct stat *buf) { int i; for(i = 0; i < exclude; i++) if((exclude_paths[i].st_dev == buf->st_dev) && (exclude_paths[i].st_ino == buf->st_ino)) return TRUE; return FALSE; } #define ADD_ENTRY(buf) \ if(exclude % EXCLUDE_SIZE == 0) { \ exclude_paths = realloc(exclude_paths, (exclude + EXCLUDE_SIZE) \ * sizeof(struct exclude_info)); \ if(exclude_paths == NULL) \ MEM_ERROR(); \ } \ exclude_paths[exclude].st_dev = buf.st_dev; \ exclude_paths[exclude++].st_ino = buf.st_ino; static int old_add_exclude(char *path) { int i; char *filename; struct stat buf; if(path[0] == '/' || strncmp(path, "./", 2) == 0 || strncmp(path, "../", 3) == 0) { if(lstat(path, &buf) == -1) { ERROR_START("Cannot stat exclude dir/file %s because " "%s", path, strerror(errno)); ERROR_EXIT(", ignoring\n"); return TRUE; } ADD_ENTRY(buf); return TRUE; } for(i = 0; i < source; i++) { int res = asprintf(&filename, "%s/%s", source_path[i], path); if(res == -1) BAD_ERROR("asprintf failed in old_add_exclude\n"); if(lstat(filename, &buf) == -1) { if(!(errno == ENOENT || errno == ENOTDIR)) { ERROR_START("Cannot stat exclude dir/file %s " "because %s", filename, strerror(errno)); ERROR_EXIT(", ignoring\n"); } free(filename); continue; } free(filename); ADD_ENTRY(buf); } return TRUE; } static void add_old_root_entry(char *name, squashfs_inode inode, unsigned int inode_number, int type) { old_root_entry = realloc(old_root_entry, sizeof(struct old_root_entry_info) * (old_root_entries + 1)); if(old_root_entry == NULL) MEM_ERROR(); old_root_entry[old_root_entries].name = strdup(name); old_root_entry[old_root_entries].inode.inode = inode; old_root_entry[old_root_entries].inode.inode_number = inode_number; old_root_entry[old_root_entries].inode.type = type; old_root_entry[old_root_entries++].inode.root_entry = TRUE; } static void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq, int freelst, char *destination_file) { int i; sigset_t sigmask, old_mask; int total_mem = readq; int reader_size; int fragment_size; int fwriter_size; /* * bwriter_size is global because it is needed in * write_file_blocks_dup() */ /* * Never allow the total size of the queues to be larger than * physical memory * * When adding together the possibly user supplied values, make * sure they've not been deliberately contrived to overflow an int */ if(add_overflow(total_mem, fragq)) BAD_ERROR("Queue sizes rediculously too large\n"); total_mem += fragq; if(add_overflow(total_mem, bwriteq)) BAD_ERROR("Queue sizes rediculously too large\n"); total_mem += bwriteq; if(add_overflow(total_mem, fwriteq)) BAD_ERROR("Queue sizes rediculously too large\n"); total_mem += fwriteq; check_usable_phys_mem(total_mem); /* * convert from queue size in Mbytes to queue size in * blocks. * * This isn't going to overflow an int unless there exists * systems with more than 8 Petabytes of RAM! */ reader_size = readq << (20 - block_log); fragment_size = fragq << (20 - block_log); bwriter_size = bwriteq << (20 - block_log); fwriter_size = fwriteq << (20 - block_log); /* * setup signal handlers for the main thread, these cleanup * deleting the destination file, if appending the * handlers for SIGTERM and SIGINT will be replaced with handlers * allowing the user to press ^C twice to restore the existing * filesystem. * * SIGUSR1 is an internal signal, which is used by the sub-threads * to tell the main thread to terminate, deleting the destination file, * or if necessary restoring the filesystem on appending */ signal(SIGTERM, sighandler); signal(SIGINT, sighandler); signal(SIGUSR1, sighandler); /* block SIGQUIT and SIGHUP, these are handled by the info thread */ sigemptyset(&sigmask); sigaddset(&sigmask, SIGQUIT); sigaddset(&sigmask, SIGHUP); if(pthread_sigmask(SIG_BLOCK, &sigmask, NULL) != 0) BAD_ERROR("Failed to set signal mask in intialise_threads\n"); /* * temporarily block these signals, so the created sub-threads * will ignore them, ensuring the main thread handles them */ sigemptyset(&sigmask); sigaddset(&sigmask, SIGINT); sigaddset(&sigmask, SIGTERM); sigaddset(&sigmask, SIGUSR1); if(pthread_sigmask(SIG_BLOCK, &sigmask, &old_mask) != 0) BAD_ERROR("Failed to set signal mask in intialise_threads\n"); if(processors == -1) { #ifdef __linux__ cpu_set_t cpu_set; CPU_ZERO(&cpu_set); if(sched_getaffinity(0, sizeof cpu_set, &cpu_set) == -1) processors = sysconf(_SC_NPROCESSORS_ONLN); else processors = CPU_COUNT(&cpu_set); #else int mib[2]; size_t len = sizeof(processors); mib[0] = CTL_HW; #ifdef HW_AVAILCPU mib[1] = HW_AVAILCPU; #else mib[1] = HW_NCPU; #endif if(sysctl(mib, 2, &processors, &len, NULL, 0) == -1) { ERROR_START("Failed to get number of available " "processors."); ERROR_EXIT(" Defaulting to 1\n"); processors = 1; } #endif } if(multiply_overflow(processors, 3) || multiply_overflow(processors * 3, sizeof(pthread_t))) BAD_ERROR("Processors too large\n"); deflator_thread = malloc(processors * 3 * sizeof(pthread_t)); if(deflator_thread == NULL) MEM_ERROR(); frag_deflator_thread = &deflator_thread[processors]; frag_thread = &frag_deflator_thread[processors]; to_reader = queue_init(1); to_deflate = queue_init(reader_size); to_process_frag = queue_init(reader_size); to_writer = queue_init(bwriter_size + fwriter_size); from_writer = queue_init(1); to_frag = queue_init(fragment_size); to_main = seq_queue_init(); if(reproducible) to_order = seq_queue_init(); else locked_fragment = queue_init(fragment_size); reader_buffer = cache_init(block_size, reader_size, 0, 0); bwriter_buffer = cache_init(block_size, bwriter_size, 1, freelst); fwriter_buffer = cache_init(block_size, fwriter_size, 1, freelst); fragment_buffer = cache_init(block_size, fragment_size, 1, 0); reserve_cache = cache_init(block_size, processors + 1, 1, 0); pthread_create(&reader_thread, NULL, reader, NULL); pthread_create(&writer_thread, NULL, writer, NULL); init_progress_bar(); init_info(); for(i = 0; i < processors; i++) { if(pthread_create(&deflator_thread[i], NULL, deflator, NULL)) BAD_ERROR("Failed to create thread\n"); if(pthread_create(&frag_deflator_thread[i], NULL, reproducible ? frag_order_deflator : frag_deflator, NULL) != 0) BAD_ERROR("Failed to create thread\n"); if(pthread_create(&frag_thread[i], NULL, frag_thrd, (void *) destination_file) != 0) BAD_ERROR("Failed to create thread\n"); } main_thread = pthread_self(); if(reproducible) pthread_create(&order_thread, NULL, frag_orderer, NULL); if(!quiet) printf("Parallel mksquashfs: Using %d processor%s\n", processors, processors == 1 ? "" : "s"); /* Restore the signal mask for the main thread */ if(pthread_sigmask(SIG_SETMASK, &old_mask, NULL) != 0) BAD_ERROR("Failed to set signal mask in intialise_threads\n"); } static long long write_inode_lookup_table() { int i, lookup_bytes = SQUASHFS_LOOKUP_BYTES(inode_count); unsigned int inode_number; void *it; if(inode_count == sinode_count) goto skip_inode_hash_table; it = realloc(inode_lookup_table, lookup_bytes); if(it == NULL) MEM_ERROR(); inode_lookup_table = it; for(i = 0; i < INODE_HASH_SIZE; i ++) { struct inode_info *inode; for(inode = inode_info[i]; inode; inode = inode->next) { inode_number = get_inode_no(inode); /* The empty action will produce orphaned inode * entries in the inode_info[] table. These * entries because they are orphaned will not be * allocated an inode number in dir_scan5(), so * skip any entries with the default dummy inode * number of 0 */ if(inode_number == 0) continue; SQUASHFS_SWAP_LONG_LONGS(&inode->inode, &inode_lookup_table[inode_number - 1], 1); } } skip_inode_hash_table: return generic_write_table(lookup_bytes, inode_lookup_table, 0, NULL, noI); } static char *get_component(char *target, char **targname) { char *start; while(*target == '/') target ++; start = target; while(*target != '/' && *target != '\0') target ++; *targname = strndup(start, target - start); while(*target == '/') target ++; return target; } static void free_path(struct pathname *paths) { int i; for(i = 0; i < paths->names; i++) { if(paths->name[i].paths) free_path(paths->name[i].paths); free(paths->name[i].name); if(paths->name[i].preg) { regfree(paths->name[i].preg); free(paths->name[i].preg); } } free(paths); } static struct pathname *add_path(struct pathname *paths, char *target, char *alltarget) { char *targname; int i, error; target = get_component(target, &targname); if(paths == NULL) { paths = malloc(sizeof(struct pathname)); if(paths == NULL) MEM_ERROR(); paths->names = 0; paths->name = NULL; } for(i = 0; i < paths->names; i++) if(strcmp(paths->name[i].name, targname) == 0) break; if(i == paths->names) { /* allocate new name entry */ paths->names ++; paths->name = realloc(paths->name, (i + 1) * sizeof(struct path_entry)); if(paths->name == NULL) MEM_ERROR(); paths->name[i].name = targname; paths->name[i].paths = NULL; if(use_regex) { paths->name[i].preg = malloc(sizeof(regex_t)); if(paths->name[i].preg == NULL) MEM_ERROR(); error = regcomp(paths->name[i].preg, targname, REG_EXTENDED|REG_NOSUB); if(error) { char str[1024]; /* overflow safe */ regerror(error, paths->name[i].preg, str, 1024); BAD_ERROR("invalid regex %s in export %s, " "because %s\n", targname, alltarget, str); } } else paths->name[i].preg = NULL; if(target[0] == '\0') /* at leaf pathname component */ paths->name[i].paths = NULL; else /* recurse adding child components */ paths->name[i].paths = add_path(NULL, target, alltarget); } else { /* existing matching entry */ free(targname); if(paths->name[i].paths == NULL) { /* No sub-directory which means this is the leaf * component of a pre-existing exclude which subsumes * the exclude currently being added, in which case stop * adding components */ } else if(target[0] == '\0') { /* at leaf pathname component and child components exist * from more specific excludes, delete as they're * subsumed by this exclude */ free_path(paths->name[i].paths); paths->name[i].paths = NULL; } else /* recurse adding child components */ add_path(paths->name[i].paths, target, alltarget); } return paths; } static void add_exclude(char *target) { if(target[0] == '/' || strncmp(target, "./", 2) == 0 || strncmp(target, "../", 3) == 0) BAD_ERROR("/, ./ and ../ prefixed excludes not supported with " "-wildcards or -regex options\n"); else if(strncmp(target, "... ", 4) == 0) stickypath = add_path(stickypath, target + 4, target + 4); else path = add_path(path, target, target); } static struct pathnames *add_subdir(struct pathnames *paths, struct pathname *path) { int count = paths == NULL ? 0 : paths->count; if(count % PATHS_ALLOC_SIZE == 0) { paths = realloc(paths, sizeof(struct pathnames) + (count + PATHS_ALLOC_SIZE) * sizeof(struct pathname *)); if(paths == NULL) MEM_ERROR(); } paths->path[count] = path; paths->count = count + 1; return paths; } static int excluded_match(char *name, struct pathname *path, struct pathnames **new) { int i; for(i = 0; i < path->names; i++) { int match = use_regex ? regexec(path->name[i].preg, name, (size_t) 0, NULL, 0) == 0 : fnmatch(path->name[i].name, name, FNM_PATHNAME|FNM_PERIOD|FNM_EXTMATCH) == 0; if(match) { if(path->name[i].paths == NULL) { /* match on a leaf component, any subdirectories * in the filesystem should be excluded */ free(*new); *new = NULL; return TRUE; } else /* match on a non-leaf component, add any * subdirectories to the new set of * subdirectories to scan for this name */ *new = add_subdir(*new, path->name[i].paths); } } return FALSE; } int excluded(char *name, struct pathnames *paths, struct pathnames **new) { int n; if(stickypath && excluded_match(name, stickypath, new)) return TRUE; for(n = 0; paths && n < paths->count; n++) { int res = excluded_match(name, paths->path[n], new); if(res) return TRUE; } /* * Either: * - no matching names found, return empty new search set, or * - one or more matches with sub-directories found (no leaf matches), * in which case return new search set. * * In either case return FALSE as we don't want to exclude this entry */ return FALSE; } static void process_exclude_file(char *argv) { FILE *fd; char buffer[MAX_LINE + 1]; /* overflow safe */ char *filename; fd = fopen(argv, "r"); if(fd == NULL) BAD_ERROR("Failed to open exclude file \"%s\" because %s\n", argv, strerror(errno)); while(fgets(filename = buffer, MAX_LINE + 1, fd) != NULL) { int len = strlen(filename); if(len == MAX_LINE && filename[len - 1] != '\n') /* line too large */ BAD_ERROR("Line too long when reading " "exclude file \"%s\", larger than %d " "bytes\n", argv, MAX_LINE); /* * Remove '\n' terminator if it exists (the last line * in the file may not be '\n' terminated) */ if(len && filename[len - 1] == '\n') filename[len - 1] = '\0'; /* Skip any leading whitespace */ while(isspace(*filename)) filename ++; /* if comment line, skip */ if(*filename == '#') continue; /* * check for initial backslash, to accommodate * filenames with leading space or leading # character */ if(*filename == '\\') filename ++; /* if line is now empty after skipping characters, skip it */ if(*filename == '\0') continue; if(old_exclude) old_add_exclude(filename); else add_exclude(filename); } if(ferror(fd)) BAD_ERROR("Reading exclude file \"%s\" failed because %s\n", argv, strerror(errno)); fclose(fd); } #define RECOVER_ID "Squashfs recovery file v1.0\n" #define RECOVER_ID_SIZE 28 static void write_recovery_data(struct squashfs_super_block *sBlk) { int recoverfd; long long res, bytes = sBlk->bytes_used - sBlk->inode_table_start; pid_t pid = getpid(); char *metadata; char header[] = RECOVER_ID; if(recover == FALSE) { if(!quiet) { printf("No recovery data option specified.\n"); printf("Skipping saving recovery file.\n\n"); } return; } if(recovery_pathname == NULL) { recovery_pathname = getenv("HOME"); if(recovery_pathname == NULL) BAD_ERROR("Could not read $HOME, use -recovery-path or -no-recovery options\n"); } res = asprintf(&recovery_file, "%s/squashfs_recovery_%s_%d", recovery_pathname, getbase(destination_file), pid); if(res == -1) MEM_ERROR(); metadata = malloc(bytes); if(metadata == NULL) MEM_ERROR(); res = read_fs_bytes(fd, sBlk->inode_table_start, bytes, metadata); if(res == 0) { ERROR("Failed to read append filesystem metadata\n"); BAD_ERROR("Filesystem corrupted?\n"); } recoverfd = open(recovery_file, O_CREAT | O_TRUNC | O_RDWR, S_IRWXU); if(recoverfd == -1) BAD_ERROR("Failed to create recovery file, because %s. " "Aborting\n", strerror(errno)); if(write_bytes(recoverfd, header, RECOVER_ID_SIZE) == -1) BAD_ERROR("Failed to write recovery file, because %s\n", strerror(errno)); if(write_bytes(recoverfd, sBlk, sizeof(struct squashfs_super_block)) == -1) BAD_ERROR("Failed to write recovery file, because %s\n", strerror(errno)); if(write_bytes(recoverfd, metadata, bytes) == -1) BAD_ERROR("Failed to write recovery file, because %s\n", strerror(errno)); res = close(recoverfd); if(res == -1) BAD_ERROR("Failed to close recovery file, close returned %s\n", strerror(errno)); free(metadata); printf("Recovery file \"%s\" written\n", recovery_file); printf("If Mksquashfs aborts abnormally (i.e. power failure), run\n"); printf("mksquashfs - %s -recover %s\n", destination_file, recovery_file); printf("to restore filesystem\n\n"); } static void read_recovery_data(char *recovery_file, char *destination_file) { int fd, recoverfd; struct squashfs_super_block orig_sBlk, sBlk; char *metadata; long long res, bytes; struct stat buf; char header[] = RECOVER_ID; char header2[RECOVER_ID_SIZE]; recoverfd = open(recovery_file, O_RDONLY); if(recoverfd == -1) BAD_ERROR("Failed to open recovery file because %s\n", strerror(errno)); if(stat(destination_file, &buf) == -1) BAD_ERROR("Failed to stat destination file, because %s\n", strerror(errno)); fd = open(destination_file, O_RDWR); if(fd == -1) BAD_ERROR("Failed to open destination file because %s\n", strerror(errno)); res = read_bytes(recoverfd, header2, RECOVER_ID_SIZE); if(res == -1) BAD_ERROR("Failed to read recovery file, because %s\n", strerror(errno)); if(res < RECOVER_ID_SIZE) BAD_ERROR("Recovery file appears to be truncated\n"); if(strncmp(header, header2, RECOVER_ID_SIZE) !=0 ) BAD_ERROR("Not a recovery file\n"); res = read_bytes(recoverfd, &sBlk, sizeof(struct squashfs_super_block)); if(res == -1) BAD_ERROR("Failed to read recovery file, because %s\n", strerror(errno)); if(res < sizeof(struct squashfs_super_block)) BAD_ERROR("Recovery file appears to be truncated\n"); res = read_fs_bytes(fd, 0, sizeof(struct squashfs_super_block), &orig_sBlk); if(res == 0) { ERROR("Failed to read superblock from output filesystem\n"); BAD_ERROR("Output filesystem is empty!\n"); } if(memcmp(((char *) &sBlk) + 4, ((char *) &orig_sBlk) + 4, sizeof(struct squashfs_super_block) - 4) != 0) BAD_ERROR("Recovery file and destination file do not seem to " "match\n"); bytes = sBlk.bytes_used - sBlk.inode_table_start; metadata = malloc(bytes); if(metadata == NULL) MEM_ERROR(); res = read_bytes(recoverfd, metadata, bytes); if(res == -1) BAD_ERROR("Failed to read recovery file, because %s\n", strerror(errno)); if(res < bytes) BAD_ERROR("Recovery file appears to be truncated\n"); write_destination(fd, 0, sizeof(struct squashfs_super_block), &sBlk); write_destination(fd, sBlk.inode_table_start, bytes, metadata); res = close(recoverfd); if(res == -1) BAD_ERROR("Failed to close recovery file, close returned %s\n", strerror(errno)); res = close(fd); if(res == -1) BAD_ERROR("Failed to close output filesystem, close returned %s\n", strerror(errno)); printf("Successfully wrote recovery file \"%s\". Exiting\n", recovery_file); exit(0); } static void write_filesystem_tables(struct squashfs_super_block *sBlk) { sBlk->fragments = fragments; sBlk->no_ids = id_count; sBlk->inode_table_start = write_inodes(); sBlk->directory_table_start = write_directories(); sBlk->fragment_table_start = write_fragment_table(); sBlk->lookup_table_start = exportable ? write_inode_lookup_table() : SQUASHFS_INVALID_BLK; sBlk->id_table_start = write_id_table(); sBlk->xattr_id_table_start = write_xattrs(); TRACE("sBlk->inode_table_start 0x%llx\n", sBlk->inode_table_start); TRACE("sBlk->directory_table_start 0x%llx\n", sBlk->directory_table_start); TRACE("sBlk->fragment_table_start 0x%llx\n", sBlk->fragment_table_start); if(exportable) TRACE("sBlk->lookup_table_start 0x%llx\n", sBlk->lookup_table_start); sBlk->bytes_used = bytes; sBlk->compression = comp->id; SQUASHFS_INSWAP_SUPER_BLOCK(sBlk); write_destination(fd, SQUASHFS_START, sizeof(*sBlk), sBlk); total_bytes += total_inode_bytes + total_directory_bytes + sizeof(struct squashfs_super_block) + total_xattr_bytes; } static int _parse_numberll(char *start, long long *res, int size, int base) { char *end; long long number; errno = 0; /* To distinguish success/failure after call */ number = strtoll(start, &end, base); /* * check for strtoll underflow or overflow in conversion, and other * errors. */ if((errno == ERANGE && (number == LLONG_MIN || number == LLONG_MAX)) || (errno != 0 && number == 0)) return 0; /* reject negative numbers as invalid */ if(number < 0) return 0; if(size) { /* * Check for multiplier and trailing junk. * But first check that a number exists before the * multiplier */ if(end == start) return 0; switch(end[0]) { case 'g': case 'G': if(multiply_overflowll(number, 1073741824)) return 0; number *= 1073741824; if(end[1] != '\0') /* trailing junk after multiplier, but * allow it to be "bytes" */ if(strcmp(end + 1, "bytes")) return 0; break; case 'm': case 'M': if(multiply_overflowll(number, 1048576)) return 0; number *= 1048576; if(end[1] != '\0') /* trailing junk after multiplier, but * allow it to be "bytes" */ if(strcmp(end + 1, "bytes")) return 0; break; case 'k': case 'K': if(multiply_overflowll(number, 1024)) return 0; number *= 1024; if(end[1] != '\0') /* trailing junk after multiplier, but * allow it to be "bytes" */ if(strcmp(end + 1, "bytes")) return 0; break; case '\0': break; default: /* trailing junk after number */ return 0; } } else if(end[0] != '\0') /* trailing junk after number */ return 0; *res = number; return 1; } static int parse_numberll(char *start, long long *res, int size) { return _parse_numberll(start, res, size, 10); } static int parse_number(char *start, int *res, int size) { long long number; if(!_parse_numberll(start, &number, size, 10)) return 0; /* check if long result will overflow signed int */ if(number > INT_MAX) return 0; *res = (int) number; return 1; } static int parse_number_unsigned(char *start, unsigned int *res, int size) { long long number; if(!_parse_numberll(start, &number, size, 10)) return 0; /* check if long result will overflow unsigned int */ if(number > UINT_MAX) return 0; *res = (unsigned int) number; return 1; } static int parse_num(char *arg, int *res) { return parse_number(arg, res, 0); } static int parse_num_unsigned(char *arg, unsigned int *res) { return parse_number_unsigned(arg, res, 0); } static int parse_mode(char *arg, mode_t *res) { long long number; if(!_parse_numberll(arg, &number, 0, 8)) return 0; if(number > 07777) return 0; *res = (mode_t) number; return 1; } static int get_physical_memory() { /* * Long longs are used here because with PAE, a 32-bit * machine can have more than 4GB of physical memory * * sysconf(_SC_PHYS_PAGES) relies on /proc being mounted. * If it fails use sysinfo, if that fails return 0 */ long long num_pages = sysconf(_SC_PHYS_PAGES); long long page_size = sysconf(_SC_PAGESIZE); int phys_mem; #ifdef __linux__ if(num_pages == -1 || page_size == -1) { struct sysinfo sys; int res = sysinfo(&sys); if(res == -1) return 0; num_pages = sys.totalram; page_size = sys.mem_unit; } #endif phys_mem = num_pages * page_size >> 20; if(phys_mem < SQUASHFS_LOWMEM) BAD_ERROR("Mksquashfs requires more physical memory than is " "available!\n"); return phys_mem; } static void check_usable_phys_mem(int total_mem) { /* * We want to allow users to use as much of their physical * memory as they wish. However, for practical reasons there are * limits which need to be imposed, to protect users from themselves * and to prevent people from using Mksquashfs as a DOS attack by using * all physical memory. Mksquashfs uses memory to cache data from disk * to optimise performance. It is pointless to ask it to use more * than 75% of physical memory, as this causes thrashing and it is thus * self-defeating. */ int mem = get_physical_memory(); mem = (mem >> 1) + (mem >> 2); /* 75% */ if(total_mem > mem && mem) { ERROR("Total memory requested is more than 75%% of physical " "memory.\n"); ERROR("Mksquashfs uses memory to cache data from disk to " "optimise performance.\n"); ERROR("It is pointless to ask it to use more than this amount " "of memory, as this\n"); ERROR("causes thrashing and it is thus self-defeating.\n"); BAD_ERROR("Requested memory size too large\n"); } if(sizeof(void *) == 4 && total_mem > 2048) { /* * If we're running on a kernel with PAE or on a 64-bit kernel, * then the 75% physical memory limit can still easily exceed * the addressable memory by this process. * * Due to the typical kernel/user-space split (1GB/3GB, or * 2GB/2GB), we have to conservatively assume the 32-bit * processes can only address 2-3GB. So refuse if the user * tries to allocate more than 2GB. */ ERROR("Total memory requested may exceed maximum " "addressable memory by this process\n"); BAD_ERROR("Requested memory size too large\n"); } } static int get_default_phys_mem() { /* * get_physical_memory() relies on /proc being mounted. * If it fails, issue a warning, and use * SQUASHFS_LOWMEM / SQUASHFS_TAKE as default, * and allow a larger value to be set with -mem. */ int mem = get_physical_memory(); if(mem == 0) { mem = SQUASHFS_LOWMEM / SQUASHFS_TAKE; ERROR("Warning: Cannot get size of physical memory, probably " "because /proc is missing.\n"); ERROR("Warning: Defaulting to minimal use of %d Mbytes, use " "-mem to set a better value,\n", mem); ERROR("Warning: or fix /proc.\n"); } else mem /= SQUASHFS_TAKE; if(sizeof(void *) == 4 && mem > 640) { /* * If we're running on a kernel with PAE or on a 64-bit kernel, * the default memory usage can exceed the addressable * memory by this process. * Due to the typical kernel/user-space split (1GB/3GB, or * 2GB/2GB), we have to conservatively assume the 32-bit * processes can only address 2-3GB. So limit the default * usage to 640M, which gives room for other data. */ mem = 640; } return mem; } static void calculate_queue_sizes(int mem, int *readq, int *fragq, int *bwriteq, int *fwriteq) { *readq = mem / SQUASHFS_READQ_MEM; *bwriteq = mem / SQUASHFS_BWRITEQ_MEM; *fwriteq = mem / SQUASHFS_FWRITEQ_MEM; *fragq = mem - *readq - *bwriteq - *fwriteq; } static void open_log_file(char *filename) { log_fd=fopen(filename, "w"); if(log_fd == NULL) BAD_ERROR("Failed to open log file \"%s\" because %s\n", filename, strerror(errno)); logging=TRUE; } static void check_env_var() { char *time_string = getenv("SOURCE_DATE_EPOCH"); unsigned int time; if(time_string != NULL) { /* * We cannot have both command-line options and environment * variable trying to set the timestamp(s) at the same * time. Semantically both are FORCE options which cannot be * over-ridden elsewhere (otherwise they can't be relied on). * * So refuse to continue if both are set. */ if(mkfs_time_opt || all_time_opt) BAD_ERROR("SOURCE_DATE_EPOCH and command line options " "can't be used at the same time to set " "timestamp(s)\n"); if(!parse_num_unsigned(time_string, &time)) { ERROR("Env Var SOURCE_DATE_EPOCH has invalid time value\n"); EXIT_MKSQUASHFS(); } all_time = mkfs_time = time; all_time_opt = mkfs_time_opt = TRUE; } } static void print_options(FILE *stream, char *name, int total_mem) { fprintf(stream, "SYNTAX:%s source1 source2 ... FILESYSTEM [OPTIONS] ", name); fprintf(stream, "[-e list of\nexclude dirs/files]\n"); fprintf(stream, "\nFilesystem compression options:\n"); fprintf(stream, "-b \t\tset data block to . Default "); fprintf(stream, "128 Kbytes.\n"); fprintf(stream, "\t\t\tOptionally a suffix of K or M can be given to "); fprintf(stream, "specify\n\t\t\tKbytes or Mbytes respectively\n"); fprintf(stream, "-comp \t\tselect compression\n"); fprintf(stream, "\t\t\tCompressors available:\n"); display_compressors(stream, "\t\t\t", COMP_DEFAULT); fprintf(stream, "-noI\t\t\tdo not compress inode table\n"); fprintf(stream, "-noId\t\t\tdo not compress the uid/gid table (implied by "); fprintf(stream, "-noI)\n"); fprintf(stream, "-noD\t\t\tdo not compress data blocks\n"); fprintf(stream, "-noF\t\t\tdo not compress fragment blocks\n"); fprintf(stream, "-noX\t\t\tdo not compress extended attributes\n"); fprintf(stream, "-no-compression\t\tdo not compress any of the data "); fprintf(stream, "or metadata. This is\n\t\t\tequivalent to "); fprintf(stream, "specifying -noI -noD -noF and -noX\n"); fprintf(stream, "\nFilesystem build options:\n"); fprintf(stream, "-tar\t\t\tread uncompressed tar file from standard in (stdin)\n"); fprintf(stream, "-no-strip\t\tact like tar, and do not strip leading "); fprintf(stream, "directories\n\t\t\tfrom source files\n"); fprintf(stream, "-tarstyle\t\talternative name for -no-strip\n"); fprintf(stream, "-cpiostyle\t\tact like cpio, and read file "); fprintf(stream, "pathnames from standard in\n\t\t\t(stdin)\n"); fprintf(stream, "-cpiostyle0\t\tlike -cpiostyle, but filenames are "); fprintf(stream, "null terminated. Can\n\t\t\tbe used with find "); fprintf(stream, "-print0 action\n"); fprintf(stream, "-reproducible\t\tbuild filesystems that are reproducible"); fprintf(stream, REP_STR "\n"); fprintf(stream, "-not-reproducible\tbuild filesystems that are not reproducible"); fprintf(stream, NOREP_STR "\n"); fprintf(stream, "-mkfs-time